Index: webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java =================================================================== --- webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java (revision 1416701) +++ webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java (working copy) @@ -19,6 +19,7 @@ import java.math.BigInteger; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -45,6 +46,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertTrue; public class TestHCatClient { @@ -101,7 +103,7 @@ String tableOne = "testTable1"; String tableTwo = "testTable2"; HCatClient client = HCatClient.create(new Configuration(hcatConf)); - client.dropDatabase(db, true, HCatClient.DROP_DB_MODE.CASCADE); + client.dropDatabase(db, true, HCatClient.DropDBMode.CASCADE); HCatCreateDBDesc dbDesc = HCatCreateDBDesc.create(db).ifNotExists(false) .build(); @@ -160,7 +162,7 @@ HCatClient client = HCatClient.create(new Configuration(hcatConf)); String dbName = "ptnDB"; String tableName = "pageView"; - client.dropDatabase(dbName, true, HCatClient.DROP_DB_MODE.CASCADE); + client.dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE); HCatCreateDBDesc dbDesc = HCatCreateDBDesc.create(dbName) .ifNotExists(true).build(); @@ -231,7 +233,7 @@ public void testDatabaseLocation() throws Exception { HCatClient client = HCatClient.create(new Configuration(hcatConf)); String dbName = "locationDB"; - client.dropDatabase(dbName, true, HCatClient.DROP_DB_MODE.CASCADE); + client.dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE); HCatCreateDBDesc dbDesc = HCatCreateDBDesc.create(dbName) .ifNotExists(true).location("/tmp/" + dbName).build(); @@ -370,4 +372,34 @@ assertTrue("The expected exception was never thrown.", isExceptionCaught); } } + + @Test + public void testUpdateTableSchema() throws Exception { + try { + HCatClient client = HCatClient.create(new Configuration(hcatConf)); + final String dbName = "testUpdateTableSchema_DBName"; + final String tableName = "testUpdateTableSchema_TableName"; + + client.dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE); + + client.createDatabase(HCatCreateDBDesc.create(dbName).build()); + List oldSchema = Arrays.asList(new HCatFieldSchema("foo", Type.INT, ""), + new HCatFieldSchema("bar", Type.STRING, "")); + client.createTable(HCatCreateTableDesc.create(dbName, tableName, oldSchema).build()); + + List newSchema = Arrays.asList(new HCatFieldSchema("completely", Type.DOUBLE, ""), + new HCatFieldSchema("new", Type.FLOAT, ""), + new HCatFieldSchema("fields", Type.STRING, "")); + + client.updateTableSchema(dbName, tableName, newSchema); + + assertArrayEquals(newSchema.toArray(), client.getTable(dbName, tableName).getCols().toArray()); + + client.dropDatabase(dbName, false, HCatClient.DropDBMode.CASCADE); + } + catch (Exception exception) { + LOG.error("Unexpected exception.", exception); + assertTrue("Unexpected exception: " + exception.getMessage(), false); + } + } } Index: webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClientHMSImpl.java =================================================================== --- webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClientHMSImpl.java (revision 1416701) +++ webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClientHMSImpl.java (working copy) @@ -44,6 +44,8 @@ import org.apache.hadoop.hive.metastore.api.UnknownTableException; import org.apache.hcatalog.common.HCatException; import org.apache.hcatalog.common.HCatUtil; +import org.apache.hcatalog.data.schema.HCatFieldSchema; +import org.apache.hcatalog.data.schema.HCatSchemaUtils; import org.apache.thrift.TException; /** @@ -58,7 +60,7 @@ @Override public List listDatabaseNamesByPattern(String pattern) - throws HCatException, ConnectionFailureException { + throws HCatException { List dbNames = null; try { dbNames = hmsClient.getDatabases(pattern); @@ -69,8 +71,7 @@ } @Override - public HCatDatabase getDatabase(String dbName) throws HCatException, - ConnectionFailureException { + public HCatDatabase getDatabase(String dbName) throws HCatException { HCatDatabase db = null; try { Database hiveDB = hmsClient.getDatabase(checkDB(dbName)); @@ -91,8 +92,7 @@ } @Override - public void createDatabase(HCatCreateDBDesc dbInfo) throws HCatException, - ConnectionFailureException { + public void createDatabase(HCatCreateDBDesc dbInfo) throws HCatException { try { hmsClient.createDatabase(dbInfo.toHiveDb()); } catch (AlreadyExistsException exp) { @@ -113,14 +113,9 @@ } @Override - public void dropDatabase(String dbName, boolean ifExists, DROP_DB_MODE mode) - throws HCatException, ConnectionFailureException { - boolean isCascade; - if (mode.toString().equalsIgnoreCase("cascade")) { - isCascade = true; - } else { - isCascade = false; - } + public void dropDatabase(String dbName, boolean ifExists, DropDBMode mode) + throws HCatException { + boolean isCascade = mode.toString().equalsIgnoreCase("cascade"); try { hmsClient.dropDatabase(checkDB(dbName), true, ifExists, isCascade); } catch (NoSuchObjectException e) { @@ -141,7 +136,7 @@ @Override public List listTableNamesByPattern(String dbName, - String tablePattern) throws HCatException, ConnectionFailureException { + String tablePattern) throws HCatException { List tableNames = null; try { tableNames = hmsClient.getTables(checkDB(dbName), tablePattern); @@ -154,7 +149,7 @@ @Override public HCatTable getTable(String dbName, String tableName) - throws HCatException, ConnectionFailureException { + throws HCatException { HCatTable table = null; try { Table hiveTable = hmsClient.getTable(checkDB(dbName), tableName); @@ -175,11 +170,11 @@ @Override public void createTable(HCatCreateTableDesc createTableDesc) - throws HCatException, ConnectionFailureException { + throws HCatException { try { hmsClient.createTable(createTableDesc.toHiveTable(hiveConfig)); } catch (AlreadyExistsException e) { - if (createTableDesc.getIfNotExists() == false) { + if (!createTableDesc.getIfNotExists()) { throw new HCatException( "AlreadyExistsException while creating table.", e); } @@ -201,9 +196,33 @@ } @Override + public void updateTableSchema(String dbName, String tableName, List columnSchema) + throws HCatException { + try { + Table table = hmsClient.getTable(dbName, tableName); + table.getSd().setCols(HCatSchemaUtils.getFieldSchemas(columnSchema)); + hmsClient.alter_table(dbName, tableName, table); + } + catch (InvalidOperationException e) { + throw new HCatException("InvalidOperationException while updating table schema.", e); + } + catch (MetaException e) { + throw new HCatException("MetaException while updating table schema.", e); + } + catch (NoSuchObjectException e) { + throw new HCatException( + "NoSuchObjectException while updating table schema.", e); + } + catch (TException e) { + throw new ConnectionFailureException( + "TException while updating table schema.", e); + } + } + + @Override public void createTableLike(String dbName, String existingTblName, String newTableName, boolean ifNotExists, boolean isExternal, - String location) throws HCatException, ConnectionFailureException { + String location) throws HCatException { Table hiveTable = getHiveTableLike(checkDB(dbName), existingTblName, newTableName, ifNotExists, location); @@ -236,7 +255,7 @@ @Override public void dropTable(String dbName, String tableName, boolean ifExists) - throws HCatException, ConnectionFailureException { + throws HCatException { try { hmsClient.dropTable(checkDB(dbName), tableName, true, ifExists); } catch (NoSuchObjectException e) { @@ -254,7 +273,7 @@ @Override public void renameTable(String dbName, String oldName, String newName) - throws HCatException, ConnectionFailureException { + throws HCatException { Table tbl; try { Table oldtbl = hmsClient.getTable(checkDB(dbName), oldName); @@ -286,7 +305,7 @@ @Override public List getPartitions(String dbName, String tblName) - throws HCatException, ConnectionFailureException { + throws HCatException { List hcatPtns = new ArrayList(); try { List hivePtns = hmsClient.listPartitions( @@ -309,8 +328,7 @@ @Override public HCatPartition getPartition(String dbName, String tableName, - Map partitionSpec) throws HCatException, - ConnectionFailureException { + Map partitionSpec) throws HCatException { HCatPartition partition = null; try { ArrayList ptnValues = new ArrayList(); @@ -335,7 +353,7 @@ @Override public void addPartition(HCatAddPartitionDesc partInfo) - throws HCatException, ConnectionFailureException { + throws HCatException { Table tbl = null; try { tbl = hmsClient.getTable(partInfo.getDatabaseName(), @@ -367,7 +385,7 @@ @Override public void dropPartition(String dbName, String tableName, Map partitionSpec, boolean ifExists) - throws HCatException, ConnectionFailureException { + throws HCatException { try { List ptnValues = new ArrayList(); ptnValues.addAll(partitionSpec.values()); @@ -389,8 +407,7 @@ @Override public List listPartitionsByFilter(String dbName, - String tblName, String filter) throws HCatException, - ConnectionFailureException { + String tblName, String filter) throws HCatException { List hcatPtns = new ArrayList(); try { List hivePtns = hmsClient.listPartitionsByFilter( @@ -414,7 +431,7 @@ @Override public void markPartitionForEvent(String dbName, String tblName, Map partKVs, PartitionEventType eventType) - throws HCatException, ConnectionFailureException { + throws HCatException { try { hmsClient.markPartitionForEvent(checkDB(dbName), tblName, partKVs, eventType); @@ -449,7 +466,7 @@ @Override public boolean isPartitionMarkedForEvent(String dbName, String tblName, Map partKVs, PartitionEventType eventType) - throws HCatException, ConnectionFailureException { + throws HCatException { boolean isMarked = false; try { isMarked = hmsClient.isPartitionMarkedForEvent(checkDB(dbName), @@ -485,8 +502,7 @@ @Override public String getDelegationToken(String owner, - String renewerKerberosPrincipalName) throws HCatException, - ConnectionFailureException { + String renewerKerberosPrincipalName) throws HCatException { String token = null; try { token = hmsClient.getDelegationToken(owner, @@ -503,8 +519,7 @@ } @Override - public long renewDelegationToken(String tokenStrForm) throws HCatException, - ConnectionFailureException { + public long renewDelegationToken(String tokenStrForm) throws HCatException { long time = 0; try { time = hmsClient.renewDelegationToken(tokenStrForm); @@ -521,7 +536,7 @@ @Override public void cancelDelegationToken(String tokenStrForm) - throws HCatException, ConnectionFailureException { + throws HCatException { try { hmsClient.cancelDelegationToken(tokenStrForm); } catch (MetaException e) { @@ -541,8 +556,7 @@ * Configuration) */ @Override - void initialize(Configuration conf) throws HCatException, - ConnectionFailureException { + void initialize(Configuration conf) throws HCatException { this.config = conf; try { hiveConfig = HCatUtil.getHiveConf(config); @@ -559,7 +573,7 @@ private Table getHiveTableLike(String dbName, String existingTblName, String newTableName, boolean isExternal, String location) - throws HCatException, ConnectionFailureException { + throws HCatException { Table oldtbl = null; Table newTable = null; try { @@ -626,7 +640,7 @@ */ @Override public int addPartitions(List partInfoList) - throws HCatException, ConnectionFailureException { + throws HCatException { int numPartitions = -1; if ((partInfoList == null) || (partInfoList.size() == 0)) { throw new HCatException("The partition list is null or empty."); Index: webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClient.java =================================================================== --- webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClient.java (revision 1417356) +++ webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClient.java (working copy) @@ -24,13 +24,14 @@ import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.metastore.api.PartitionEventType; import org.apache.hcatalog.common.HCatException; +import org.apache.hcatalog.data.schema.HCatFieldSchema; /** * The abstract class HCatClient containing APIs for HCatalog DDL commands. */ public abstract class HCatClient { - public enum DROP_DB_MODE {RESTRICT, CASCADE} + public enum DropDBMode {RESTRICT, CASCADE} public static final String HCAT_CLIENT_IMPL_CLASS = "hcat.client.impl.class"; @@ -39,10 +40,9 @@ * * @param conf An instance of configuration. * @return An instance of HCatClient. - * @throws HCatException,ConnectionFailureException + * @throws HCatException */ - public static HCatClient create(Configuration conf) throws HCatException, - ConnectionFailureException { + public static HCatClient create(Configuration conf) throws HCatException { HCatClient client = null; String className = conf.get(HCAT_CLIENT_IMPL_CLASS, HCatClientHMSImpl.class.getName()); @@ -67,7 +67,7 @@ return client; } - abstract void initialize(Configuration conf) throws HCatException, ConnectionFailureException; + abstract void initialize(Configuration conf) throws HCatException; /** * Get all existing databases that match the given @@ -75,28 +75,28 @@ * * @param pattern java re pattern * @return list of database names - * @throws HCatException,ConnectionFailureException + * @throws HCatException */ public abstract List listDatabaseNamesByPattern(String pattern) - throws HCatException, ConnectionFailureException; + throws HCatException; /** * Gets the database. * * @param dbName The name of the database. * @return An instance of HCatDatabaseInfo. - * @throws HCatException,ConnectionFailureException + * @throws HCatException */ - public abstract HCatDatabase getDatabase(String dbName) throws HCatException, ConnectionFailureException; + public abstract HCatDatabase getDatabase(String dbName) throws HCatException; /** * Creates the database. * * @param dbInfo An instance of HCatCreateDBDesc. - * @throws HCatException,ConnectionFailureException + * @throws HCatException */ public abstract void createDatabase(HCatCreateDBDesc dbInfo) - throws HCatException, ConnectionFailureException; + throws HCatException; /** * Drops a database. @@ -107,21 +107,21 @@ * @param mode This is set to either "restrict" or "cascade". Restrict will * remove the schema if all the tables are empty. Cascade removes * everything including data and definitions. - * @throws HCatException,ConnectionFailureException + * @throws HCatException */ public abstract void dropDatabase(String dbName, boolean ifExists, - DROP_DB_MODE mode) throws HCatException, ConnectionFailureException; + DropDBMode mode) throws HCatException; /** * Returns all existing tables from the specified database which match the given * pattern. The matching occurs as per Java regular expressions. - * @param dbName - * @param tablePattern + * @param dbName The name of the DB (to be searched) + * @param tablePattern The regex for the table-name * @return list of table names - * @throws HCatException,ConnectionFailureException + * @throws HCatException */ public abstract List listTableNamesByPattern(String dbName, String tablePattern) - throws HCatException, ConnectionFailureException; + throws HCatException; /** * Gets the table. @@ -129,21 +129,31 @@ * @param dbName The name of the database. * @param tableName The name of the table. * @return An instance of HCatTableInfo. - * @throws HCatException,ConnectionFailureException + * @throws HCatException */ public abstract HCatTable getTable(String dbName, String tableName) - throws HCatException, ConnectionFailureException; + throws HCatException; /** * Creates the table. * * @param createTableDesc An instance of HCatCreateTableDesc class. - * @throws HCatException,ConnectionFailureException + * @throws HCatException */ - public abstract void createTable(HCatCreateTableDesc createTableDesc) - throws HCatException, ConnectionFailureException; + public abstract void createTable(HCatCreateTableDesc createTableDesc) throws HCatException; /** + * Updates the Table's column schema to the specified definition. + * + * @param dbName The name of the database. + * @param tableName The name of the table. + * @param columnSchema The (new) definition of the column schema (i.e. list of fields). + * + */ + public abstract void updateTableSchema(String dbName, String tableName, List columnSchema) + throws HCatException; + + /** * Creates the table like an existing table. * * @param dbName The name of the database. @@ -153,11 +163,11 @@ * @param isExternal Set to "true", if table has be created at a different * location other than default. * @param location The location for the table. - * @throws HCatException,ConnectionFailureException + * @throws HCatException */ public abstract void createTableLike(String dbName, String existingTblName, String newTableName, boolean ifNotExists, boolean isExternal, - String location) throws HCatException, ConnectionFailureException; + String location) throws HCatException; /** * Drop table. @@ -166,10 +176,10 @@ * @param tableName The name of the table. * @param ifExists Hive returns an error if the database specified does not exist, * unless ifExists is set to true. - * @throws HCatException,ConnectionFailureException + * @throws HCatException */ public abstract void dropTable(String dbName, String tableName, - boolean ifExists) throws HCatException, ConnectionFailureException; + boolean ifExists) throws HCatException; /** * Renames a table. @@ -177,10 +187,10 @@ * @param dbName The name of the database. * @param oldName The name of the table to be renamed. * @param newName The new name of the table. - * @throws HCatException,ConnectionFailureException + * @throws HCatException */ public abstract void renameTable(String dbName, String oldName, - String newName) throws HCatException, ConnectionFailureException; + String newName) throws HCatException; /** * Gets all the partitions. @@ -188,10 +198,10 @@ * @param dbName The name of the database. * @param tblName The name of the table. * @return A list of partitions. - * @throws HCatException,ConnectionFailureException + * @throws HCatException */ public abstract List getPartitions(String dbName, String tblName) - throws HCatException, ConnectionFailureException; + throws HCatException; /** * Gets the partition. @@ -200,29 +210,29 @@ * @param tableName The table name. * @param partitionSpec The partition specification, {[col_name,value],[col_name2,value2]}. * @return An instance of HCatPartitionInfo. - * @throws HCatException,ConnectionFailureException + * @throws HCatException */ public abstract HCatPartition getPartition(String dbName, String tableName, - Map partitionSpec) throws HCatException, ConnectionFailureException; + Map partitionSpec) throws HCatException; /** * Adds the partition. * * @param partInfo An instance of HCatAddPartitionDesc. - * @throws HCatException,ConnectionFailureException + * @throws HCatException */ public abstract void addPartition(HCatAddPartitionDesc partInfo) - throws HCatException, ConnectionFailureException; + throws HCatException; /** * Adds a list of partitions. * * @param partInfoList A list of HCatAddPartitionDesc. * @return The number of partitions added. - * @throws HCatException,ConnectionFailureException + * @throws HCatException */ public abstract int addPartitions(List partInfoList) - throws HCatException, ConnectionFailureException; + throws HCatException; /** * Drops partition. @@ -235,7 +245,7 @@ */ public abstract void dropPartition(String dbName, String tableName, Map partitionSpec, boolean ifExists) - throws HCatException, ConnectionFailureException; + throws HCatException; /** * List partitions by filter. @@ -246,10 +256,10 @@ * for example "part1 = \"p1_abc\" and part2 <= "\p2_test\"". Filtering can * be done only on string partition keys. * @return list of partitions - * @throws HCatException,ConnectionFailureException + * @throws HCatException */ public abstract List listPartitionsByFilter(String dbName, String tblName, - String filter) throws HCatException, ConnectionFailureException; + String filter) throws HCatException; /** * Mark partition for event. @@ -258,11 +268,11 @@ * @param tblName The table name. * @param partKVs the key-values associated with the partition. * @param eventType the event type - * @throws HCatException,ConnectionFailureException + * @throws HCatException */ public abstract void markPartitionForEvent(String dbName, String tblName, Map partKVs, PartitionEventType eventType) - throws HCatException, ConnectionFailureException; + throws HCatException; /** * Checks if a partition is marked for event. @@ -272,11 +282,11 @@ * @param partKVs the key-values associated with the partition. * @param eventType the event type * @return true, if is partition marked for event - * @throws HCatException,ConnectionFailureException + * @throws HCatException */ public abstract boolean isPartitionMarkedForEvent(String dbName, String tblName, Map partKVs, PartitionEventType eventType) - throws HCatException, ConnectionFailureException; + throws HCatException; /** * Gets the delegation token. @@ -287,27 +297,26 @@ * @throws HCatException,ConnectionFailureException */ public abstract String getDelegationToken(String owner, - String renewerKerberosPrincipalName) throws HCatException, - ConnectionFailureException; + String renewerKerberosPrincipalName) throws HCatException; /** * Renew delegation token. * * @param tokenStrForm the token string * @return the new expiration time - * @throws HCatException,ConnectionFailureException + * @throws HCatException */ public abstract long renewDelegationToken(String tokenStrForm) - throws HCatException, ConnectionFailureException; + throws HCatException; /** * Cancel delegation token. * * @param tokenStrForm the token string - * @throws HCatException,ConnectionFailureException + * @throws HCatException */ public abstract void cancelDelegationToken(String tokenStrForm) - throws HCatException, ConnectionFailureException; + throws HCatException; /** * Close the hcatalog client.