diff --git a/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatAddPartitionDesc.java b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatAddPartitionDesc.java index afa3054..c667382 100644 --- a/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatAddPartitionDesc.java +++ b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatAddPartitionDesc.java @@ -18,18 +18,8 @@ */ package org.apache.hive.hcatalog.api; -import java.util.ArrayList; -import java.util.List; import java.util.Map; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; -import org.apache.hadoop.hive.metastore.Warehouse; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.StorageDescriptor; -import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hive.hcatalog.common.HCatException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,16 +30,33 @@ public class HCatAddPartitionDesc { private static final Logger LOG = LoggerFactory.getLogger(HCatAddPartitionDesc.class); - private String tableName; - private String dbName; - private String location; - private Map partSpec; + private HCatPartition hcatPartition; - private HCatAddPartitionDesc(String dbName, String tbl, String loc, Map spec) { + // The following data members are only required to support the deprecated constructor (and builder). + String dbName, tableName, location; + Map partitionKeyValues; + + private HCatAddPartitionDesc(HCatPartition hcatPartition) { + this.hcatPartition = hcatPartition; + } + + private HCatAddPartitionDesc(String dbName, String tableName, String location, Map partitionKeyValues) { + this.hcatPartition = null; this.dbName = dbName; - this.tableName = tbl; - this.location = loc; - this.partSpec = spec; + this.tableName = tableName; + this.location = location; + this.partitionKeyValues = partitionKeyValues; + } + + HCatPartition getHCatPartition() { + return hcatPartition; + } + + HCatPartition getHCatPartition(HCatTable hcatTable) throws HCatException { + assert hcatPartition == null : "hcatPartition should have been null at this point."; + assert dbName.equalsIgnoreCase(hcatTable.getDbName()) : "DB names don't match."; + assert tableName.equalsIgnoreCase(hcatTable.getTableName()) : "Table names don't match."; + return new HCatPartition(hcatTable, partitionKeyValues, location); } /** @@ -57,18 +64,19 @@ private HCatAddPartitionDesc(String dbName, String tbl, String loc, Map getPartitionSpec() { - return this.partSpec; + return hcatPartition == null? partitionKeyValues : hcatPartition.getPartitionKeyValMap(); } /** @@ -76,8 +84,9 @@ public String getLocation() { * * @return the table name */ + @Deprecated // @deprecated in favour of {@link HCatPartition.#getTableName()}. To be removed in Hive 0.16. public String getTableName() { - return this.tableName; + return hcatPartition == null? tableName : hcatPartition.getTableName(); } /** @@ -85,17 +94,14 @@ public String getTableName() { * * @return the database name */ + @Deprecated // @deprecated in favour of {@link HCatPartition.#getDatabaseName()}. To be removed in Hive 0.16. public String getDatabaseName() { - return this.dbName; + return hcatPartition == null? dbName : hcatPartition.getDatabaseName(); } @Override public String toString() { - return "HCatAddPartitionDesc [" - + (tableName != null ? "tableName=" + tableName + ", " : "tableName=null") - + (dbName != null ? "dbName=" + dbName + ", " : "dbName=null") - + (location != null ? "location=" + location + ", " : "location=null") - + (partSpec != null ? "partSpec=" + partSpec : "partSpec=null") + "]"; + return "HCatAddPartitionDesc [" + hcatPartition + "]"; } /** @@ -108,61 +114,48 @@ public String toString() { * @return the builder * @throws HCatException */ - public static Builder create(String dbName, String tableName, String location, - Map partSpec) throws HCatException { + @Deprecated // @deprecated in favour of {@link HCatAddPartitionDesc.#create(HCatPartition)}. To be removed in Hive 0.16. + public static Builder create(String dbName, + String tableName, + String location, + Map partSpec + ) throws HCatException { + LOG.error("Unsupported! HCatAddPartitionDesc requires HCatTable to be specified explicitly."); return new Builder(dbName, tableName, location, partSpec); } - Partition toHivePartition(Table hiveTable) throws HCatException { - Partition hivePtn = new Partition(); - hivePtn.setDbName(this.dbName); - hivePtn.setTableName(this.tableName); - - List pvals = new ArrayList(); - for (FieldSchema field : hiveTable.getPartitionKeys()) { - String val = partSpec.get(field.getName()); - if (val == null || val.length() == 0) { - throw new HCatException("create partition: Value for key " - + field.getName() + " is null or empty"); - } - pvals.add(val); - } - - hivePtn.setValues(pvals); - StorageDescriptor sd = new StorageDescriptor(hiveTable.getSd()); - hivePtn.setSd(sd); - hivePtn.setParameters(hiveTable.getParameters()); - if (this.location != null) { - hivePtn.getSd().setLocation(this.location); - } else { - String partName; - try { - partName = Warehouse.makePartName( - hiveTable.getPartitionKeys(), pvals); - LOG.info("Setting partition location to :" + partName); - } catch (MetaException e) { - throw new HCatException("Exception while creating partition name.", e); - } - Path partPath = new Path(hiveTable.getSd().getLocation(), partName); - hivePtn.getSd().setLocation(partPath.toString()); - } - hivePtn.setCreateTime((int) (System.currentTimeMillis() / 1000)); - hivePtn.setLastAccessTimeIsSet(false); - return hivePtn; + /** + * Constructs a Builder instance, using an HCatPartition object. + * @param partition An HCatPartition instance. + * @return A Builder object that can build an appropriate HCatAddPartitionDesc. + * @throws HCatException + */ + public static Builder create(HCatPartition partition) throws HCatException { + return new Builder(partition); } + /** + * Builder class for constructing an HCatAddPartition instance. + */ public static class Builder { - private String tableName; - private String location; - private Map values; - private String dbName; + private HCatPartition hcatPartition; + + // The following data members are only required to support the deprecated constructor (and builder). + String dbName, tableName, location; + Map partitionSpec; + + private Builder(HCatPartition hcatPartition) { + this.hcatPartition = hcatPartition; + } - private Builder(String dbName, String tableName, String location, Map values) { + @Deprecated // To be removed in Hive 0.16. + private Builder(String dbName, String tableName, String location, Map partitionSpec) { + this.hcatPartition = null; this.dbName = dbName; this.tableName = tableName; this.location = location; - this.values = values; + this.partitionSpec = partitionSpec; } /** @@ -172,13 +165,9 @@ private Builder(String dbName, String tableName, String location, Map serializePartitions(List hcatPartitions) throws HCatException; + + /** + * Deserializer for an HCatPartition. + * @param hcatPartitionStringRep The String representation of the HCatPartition, presumably retrieved from {@link #serializePartition(HCatPartition)} + * @return HCatPartition instance reconstructed from the string. + * @throws HCatException, on failure to deserialze. + */ + public abstract HCatPartition deserializePartition(String hcatPartitionStringRep) throws HCatException; + + /** + * Deserializer for a list of HCatPartition strings. + * @param hcatPartitionStringReps The list of HCatPartition strings to be deserialized. + * @return A list of HCatPartition instances, each reconstructed from an entry in the string-list. + * @throws HCatException, on failure to deserialize. + */ + public abstract List deserializePartitions(List hcatPartitionStringReps) throws HCatException; + + /** * Creates the table like an existing table. * * @param dbName The name of the database. diff --git a/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java index c4b5971..b3afa72 100644 --- a/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java +++ b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java @@ -172,7 +172,7 @@ public HCatTable getTable(String dbName, String tableName) public void createTable(HCatCreateTableDesc createTableDesc) throws HCatException { try { - hmsClient.createTable(createTableDesc.toHiveTable(hiveConfig)); + hmsClient.createTable(createTableDesc.getHCatTable().toHiveTable()); } catch (AlreadyExistsException e) { if (!createTableDesc.getIfNotExists()) { throw new HCatException( @@ -220,6 +220,27 @@ public void updateTableSchema(String dbName, String tableName, List hcatPtns = new ArrayList(); try { + Table table = hmsClient.getTable(dbName, tblName); + HCatTable hcatTable = new HCatTable(table); List hivePtns = hmsClient.listPartitions( checkDB(dbName), tblName, (short) -1); for (Partition ptn : hivePtns) { - hcatPtns.add(new HCatPartition(ptn)); + hcatPtns.add(new HCatPartition(hcatTable, ptn)); } } catch (NoSuchObjectException e) { throw new ObjectNotFoundException( @@ -351,7 +374,8 @@ public HCatPartition getPartition(String dbName, String tableName, Map partitionSpec) throws HCatException { HCatPartition partition = null; try { - List partitionColumns = getTable(checkDB(dbName), tableName).getPartCols(); + HCatTable hcatTable = getTable(checkDB(dbName), tableName); + List partitionColumns = hcatTable.getPartCols(); if (partitionColumns.size() != partitionSpec.size()) { throw new HCatException("Partition-spec doesn't have the right number of partition keys."); } @@ -369,7 +393,7 @@ public HCatPartition getPartition(String dbName, String tableName, Partition hivePartition = hmsClient.getPartition(checkDB(dbName), tableName, ptnValues); if (hivePartition != null) { - partition = new HCatPartition(hivePartition); + partition = new HCatPartition(hcatTable, hivePartition); } } catch (MetaException e) { throw new HCatException( @@ -397,7 +421,17 @@ public void addPartition(HCatAddPartitionDesc partInfo) + " is not partitioned."); } - hmsClient.add_partition(partInfo.toHivePartition(tbl)); + HCatTable hcatTable = new HCatTable(tbl); + + HCatPartition hcatPartition = partInfo.getHCatPartition(); + + // TODO: Remove in Hive 0.16. + // This is only required to support the deprecated methods in HCatAddPartitionDesc.Builder. + if (hcatPartition == null) { + hcatPartition = partInfo.getHCatPartition(hcatTable); + } + + hmsClient.add_partition(hcatPartition.toHivePartition()); } catch (InvalidObjectException e) { throw new HCatException( "InvalidObjectException while adding partition.", e); @@ -458,10 +492,11 @@ private void dropPartition(Partition partition, boolean ifExists) String tblName, String filter) throws HCatException { List hcatPtns = new ArrayList(); try { + HCatTable table = getTable(dbName, tblName); List hivePtns = hmsClient.listPartitionsByFilter( checkDB(dbName), tblName, filter, (short) -1); for (Partition ptn : hivePtns) { - hcatPtns.add(new HCatPartition(ptn)); + hcatPtns.add(new HCatPartition(table, ptn)); } } catch (MetaException e) { throw new HCatException("MetaException while fetching partitions.", @@ -682,9 +717,18 @@ public int addPartitions(List partInfoList) try { tbl = hmsClient.getTable(partInfoList.get(0).getDatabaseName(), partInfoList.get(0).getTableName()); + HCatTable hcatTable = new HCatTable(tbl); ArrayList ptnList = new ArrayList(); for (HCatAddPartitionDesc desc : partInfoList) { - ptnList.add(desc.toHivePartition(tbl)); + HCatPartition hCatPartition = desc.getHCatPartition(); + + // TODO: Remove in Hive 0.16. + // This is required only to support the deprecated HCatAddPartitionDesc.Builder interfaces. + if (hCatPartition == null) { + hCatPartition = desc.getHCatPartition(hcatTable); + } + + ptnList.add(hCatPartition.toHivePartition()); } numPartitions = hmsClient.add_partitions(ptnList); } catch (InvalidObjectException e) { @@ -720,4 +764,65 @@ public String getMessageBusTopicName(String dbName, String tableName) throws HCa "TException while retrieving JMS Topic name.", e); } } + + @Override + public String serializeTable(HCatTable hcatTable) throws HCatException { + return MetadataSerializer.get().serializeTable(hcatTable); + } + + @Override + public HCatTable deserializeTable(String hcatTableStringRep) throws HCatException { + return MetadataSerializer.get().deserializeTable(hcatTableStringRep); + } + + @Override + public String serializePartition(HCatPartition hcatPartition) throws HCatException { + return MetadataSerializer.get().serializePartition(hcatPartition); + } + + @Override + public List serializePartitions(List hcatPartitions) throws HCatException { + List partStrings = new ArrayList(hcatPartitions.size()); + MetadataSerializer serializer = MetadataSerializer.get(); + + for (HCatPartition partition : hcatPartitions) { + partStrings.add(serializer.serializePartition(partition)); + } + + return partStrings; + } + + @Override + public HCatPartition deserializePartition(String hcatPartitionStringRep) throws HCatException { + HCatPartition hcatPartition = MetadataSerializer.get().deserializePartition(hcatPartitionStringRep); + hcatPartition.hcatTable(getTable(hcatPartition.getDatabaseName(), hcatPartition.getTableName())); + return hcatPartition; + } + + @Override + public List deserializePartitions(List hcatPartitionStringReps) throws HCatException { + List partitions = new ArrayList(hcatPartitionStringReps.size()); + MetadataSerializer deserializer = MetadataSerializer.get(); + HCatTable table = null; + for (String partString : hcatPartitionStringReps) { + HCatPartition partition; + if (table == null) { + partition = deserializePartition(partString); + table = partition.hcatTable(); + } + else { + partition = deserializer.deserializePartition(partString); + if (partition.getDatabaseName().equals(table.getDbName()) + && partition.getTableName().equals(table.getTableName())) { + partition.hcatTable(table); + } + else { + throw new HCatException("All partitions are not of the same table: " + + table.getDbName() + "." + table.getTableName()); + } + } + partitions.add(partition); + } + return partitions; + } } diff --git a/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatCreateTableDesc.java b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatCreateTableDesc.java index d6e9753..5293f7b 100644 --- a/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatCreateTableDesc.java +++ b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatCreateTableDesc.java @@ -19,38 +19,13 @@ package org.apache.hive.hcatalog.api; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; -import org.apache.hadoop.hive.metastore.TableType; -import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Order; -import org.apache.hadoop.hive.metastore.api.SerDeInfo; -import org.apache.hadoop.hive.metastore.api.StorageDescriptor; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; -import org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat; -import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; -import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat; -import org.apache.hadoop.hive.ql.io.orc.OrcSerde; -import org.apache.hadoop.hive.ql.io.RCFileInputFormat; -import org.apache.hadoop.hive.ql.io.RCFileOutputFormat; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; -import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; -import org.apache.hadoop.mapred.SequenceFileInputFormat; -import org.apache.hadoop.mapred.TextInputFormat; import org.apache.hive.hcatalog.common.HCatException; import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; -import org.apache.hive.hcatalog.data.schema.HCatSchemaUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * The Class HCatCreateTableDesc for defining attributes for a new table. @@ -58,31 +33,12 @@ @SuppressWarnings("deprecation") public class HCatCreateTableDesc { - private static final Logger LOG = LoggerFactory.getLogger(HCatCreateTableDesc.class); - - private String tableName; - private String dbName; - private boolean isExternal; - private String comment; - private String location; - private List cols; - private List partCols; - private List bucketCols; - private int numBuckets; - private List sortCols; - private Map tblProps; private boolean ifNotExists; - private String fileFormat; - private String inputformat; - private String outputformat; - private String serde; - private String storageHandler; - private Map serdeParams; - - private HCatCreateTableDesc(String dbName, String tableName, List columns) { - this.dbName = dbName; - this.tableName = tableName; - this.cols = columns; + private HCatTable hcatTable; + + private HCatCreateTableDesc(HCatTable hcatTable, boolean ifNotExists) { + this.hcatTable = hcatTable; + this.ifNotExists = ifNotExists; } /** @@ -93,109 +49,36 @@ private HCatCreateTableDesc(String dbName, String tableName, List columns) { return new Builder(dbName, tableName, columns); } - Table toHiveTable(HiveConf conf) throws HCatException { - - /* - * get the same defaults as are set when a Table is created via the Hive Driver. - */ - Table newTable = org.apache.hadoop.hive.ql.metadata.Table.getEmptyTable(dbName, tableName); - newTable.setDbName(dbName); - newTable.setTableName(tableName); - if (tblProps != null) { - for ( Map.Entry e : tblProps.entrySet()){ - newTable.getParameters().put(e.getKey(), e.getValue()); - } - } - - if (isExternal) { - newTable.putToParameters("EXTERNAL", "TRUE"); - newTable.setTableType(TableType.EXTERNAL_TABLE.toString()); - } else { - newTable.setTableType(TableType.MANAGED_TABLE.toString()); - } - - // Initialize an sd if one does not exist - if (newTable.getSd() == null) { - newTable.setSd(new StorageDescriptor()); - } - StorageDescriptor sd = newTable.getSd(); - - if (sd.getSerdeInfo() == null){ - sd.setSerdeInfo(new SerDeInfo()); - } - if (location != null) { - sd.setLocation(location); - } - if (this.comment != null) { - newTable.putToParameters("comment", comment); - } - if (!StringUtils.isEmpty(fileFormat)) { - sd.setInputFormat(inputformat); - sd.setOutputFormat(outputformat); - if (serde != null) { - sd.getSerdeInfo().setSerializationLib(serde); - } else { - LOG.info("Using LazySimpleSerDe for table " + tableName); - sd.getSerdeInfo() - .setSerializationLib( - org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class - .getName()); - } - } else { - try { - LOG.info("Creating instance of storage handler to get input/output, serder info."); - HiveStorageHandler sh = HiveUtils.getStorageHandler(conf, - storageHandler); - sd.setInputFormat(sh.getInputFormatClass().getName()); - sd.setOutputFormat(sh.getOutputFormatClass().getName()); - sd.getSerdeInfo().setSerializationLib( - sh.getSerDeClass().getName()); - newTable.putToParameters( - org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE, - storageHandler); - } catch (HiveException e) { - throw new HCatException( - "Exception while creating instance of storage handler", - e); - } - } - if(serdeParams != null) { - for(Map.Entry param : serdeParams.entrySet()) { - sd.getSerdeInfo().putToParameters(param.getKey(), param.getValue()); - } - } - if (this.partCols != null) { - ArrayList hivePtnCols = new ArrayList(); - for (HCatFieldSchema fs : this.partCols) { - hivePtnCols.add(HCatSchemaUtils.getFieldSchema(fs)); - } - newTable.setPartitionKeys(hivePtnCols); - } - - if (this.cols != null) { - ArrayList hiveTblCols = new ArrayList(); - for (HCatFieldSchema fs : this.cols) { - hiveTblCols.add(HCatSchemaUtils.getFieldSchema(fs)); - } - newTable.getSd().setCols(hiveTblCols); - } - - if (this.bucketCols != null) { - newTable.getSd().setBucketCols(bucketCols); - newTable.getSd().setNumBuckets(numBuckets); - } + /** + * Getter for HCatCreateTableDesc.Builder instance. + * @param table Spec for HCatTable to be created. + * @param ifNotExists Only create the table if it doesn't already exist. + * @return Builder instance. + */ + public static Builder create(HCatTable table, boolean ifNotExists) { + return new Builder(table, ifNotExists); + } - if (this.sortCols != null) { - newTable.getSd().setSortCols(sortCols); - } + /** + * Getter for HCatCreateTableDesc.Builder instance. By default, ifNotExists is false. + * So the attempt to create the table is made even if the table already exists. + * @param table Spec for HCatTable to be created. + * @return Builder instance. + */ + public static Builder create(HCatTable table) { + return new Builder(table, false); + } - newTable.setCreateTime((int) (System.currentTimeMillis() / 1000)); - newTable.setLastAccessTimeIsSet(false); - return newTable; + /** + * Getter for underlying HCatTable instance. + */ + public HCatTable getHCatTable() { + return this.hcatTable; } /** @@ -212,8 +95,9 @@ public boolean getIfNotExists() { * * @return the table name */ + @Deprecated // @deprecated in favour of {@link HCatTable.#getTableName()}. To be removed in Hive 0.16. public String getTableName() { - return this.tableName; + return this.hcatTable.getTableName(); } /** @@ -221,8 +105,9 @@ public String getTableName() { * * @return the cols */ + @Deprecated // @deprecated in favour of {@link HCatTable.#getCols()}. To be removed in Hive 0.16. public List getCols() { - return this.cols; + return this.hcatTable.getCols(); } /** @@ -230,8 +115,9 @@ public String getTableName() { * * @return the partition cols */ + @Deprecated // @deprecated in favour of {@link HCatTable.#getPartCols()}. To be removed in Hive 0.16. public List getPartitionCols() { - return this.partCols; + return this.hcatTable.getPartCols(); } /** @@ -239,12 +125,14 @@ public String getTableName() { * * @return the bucket cols */ + @Deprecated // @deprecated in favour of {@link HCatTable.#getBucketCols()}. To be removed in Hive 0.16. public List getBucketCols() { - return this.bucketCols; + return this.hcatTable.getBucketCols(); } + @Deprecated // @deprecated in favour of {@link HCatTable.#getNumBuckets()}. public int getNumBuckets() { - return this.numBuckets; + return this.hcatTable.getNumBuckets(); } /** @@ -252,8 +140,9 @@ public int getNumBuckets() { * * @return the comments */ + @Deprecated // @deprecated in favour of {@link HCatTable.#comment()}. To be removed in Hive 0.16. public String getComments() { - return this.comment; + return this.hcatTable.comment(); } /** @@ -261,8 +150,9 @@ public String getComments() { * * @return the storage handler */ + @Deprecated // @deprecated in favour of {@link HCatTable.#getStorageHandler()}. To be removed in Hive 0.16. public String getStorageHandler() { - return this.storageHandler; + return this.hcatTable.getStorageHandler(); } /** @@ -270,8 +160,9 @@ public String getStorageHandler() { * * @return the location */ + @Deprecated // @deprecated in favour of {@link HCatTable.#getLocation()}. To be removed in Hive 0.16. public String getLocation() { - return this.location; + return this.hcatTable.getLocation(); } /** @@ -279,8 +170,11 @@ public String getLocation() { * * @return the external */ + @Deprecated // @deprecated in favour of {@link HCatTable.#getTableType()}. To be removed in Hive 0.16. public boolean getExternal() { - return this.isExternal; + + return this.hcatTable.getTabletype() + .equalsIgnoreCase(HCatTable.Type.EXTERNAL_TABLE.toString()); } /** @@ -288,8 +182,9 @@ public boolean getExternal() { * * @return the sort cols */ + @Deprecated // @deprecated in favour of {@link HCatTable.#getSortCols()}. To be removed in Hive 0.16. public List getSortCols() { - return this.sortCols; + return this.hcatTable.getSortCols(); } /** @@ -297,8 +192,9 @@ public boolean getExternal() { * * @return the tbl props */ + @Deprecated // @deprecated in favour of {@link HCatTable.#getTblProps()}. To be removed in Hive 0.16. public Map getTblProps() { - return this.tblProps; + return this.hcatTable.getTblProps(); } /** @@ -306,8 +202,9 @@ public boolean getExternal() { * * @return the file format */ + @Deprecated // @deprecated in favour of {@link HCatTable.#fileFormat()}. To be removed in Hive 0.16. public String getFileFormat() { - return this.fileFormat; + return this.hcatTable.fileFormat(); } /** @@ -315,74 +212,39 @@ public String getFileFormat() { * * @return the database name */ + @Deprecated // @deprecated in favour of {@link HCatTable.#getDbName()}. To be removed in Hive 0.16. public String getDatabaseName() { - return this.dbName; + return this.hcatTable.getDbName(); } + /** * Gets the SerDe parameters; for example see {@link org.apache.hive.hcatalog.api.HCatCreateTableDesc.Builder#fieldsTerminatedBy(char)} */ + @Deprecated public Map getSerdeParams() { - return serdeParams; + return this.hcatTable.getSerdeParams(); } @Override public String toString() { - return "HCatCreateTableDesc [" - + (tableName != null ? "tableName=" + tableName + ", " : "tableName=null") - + (dbName != null ? "dbName=" + dbName + ", " : "dbName=null") - + "isExternal=" - + isExternal - + ", " - + (comment != null ? "comment=" + comment + ", " : "comment=null") - + (location != null ? "location=" + location + ", " : "location=null") - + (cols != null ? "cols=" + cols + ", " : "cols=null") - + (partCols != null ? "partCols=" + partCols + ", " : "partCols=null") - + (bucketCols != null ? "bucketCols=" + bucketCols + ", " : "bucketCols=null") - + "numBuckets=" - + numBuckets - + ", " - + (sortCols != null ? "sortCols=" + sortCols + ", " : "sortCols=null") - + (tblProps != null ? "tblProps=" + tblProps + ", " : "tblProps=null") - + "ifNotExists=" - + ifNotExists - + ", " - + (fileFormat != null ? "fileFormat=" + fileFormat + ", " : "fileFormat=null") - + (inputformat != null ? "inputformat=" + inputformat + ", " - : "inputformat=null") - + (outputformat != null ? "outputformat=" + outputformat + ", " - : "outputformat=null") - + (serde != null ? "serde=" + serde + ", " : "serde=null") - + (storageHandler != null ? "storageHandler=" + storageHandler - : "storageHandler=null") - + ",serdeParams=" + (serdeParams == null ? "null" : serdeParams) - + "]"; + return "HCatCreateTableDesc [ " + hcatTable.toString() + + ", ifNotExists = " + ifNotExists + "]"; } public static class Builder { - private String tableName; - private boolean isExternal; - private List cols; - private List partCols; - private List bucketCols; - private List sortCols; - private int numBuckets; - private String comment; - private String fileFormat; - private String location; - private String storageHandler; - private Map tblProps; private boolean ifNotExists; - private String dbName; - private Map serdeParams; - + private HCatTable hcatTable; + @Deprecated // @deprecated in favour of {@link #Builder(HCatTable, boolean)}. To be removed in Hive 0.16. private Builder(String dbName, String tableName, List columns) { - this.dbName = dbName; - this.tableName = tableName; - this.cols = columns; + hcatTable = new HCatTable(dbName, tableName).cols(columns); } + private Builder(HCatTable hcatTable, boolean ifNotExists) { + this.hcatTable = hcatTable; + this.ifNotExists = ifNotExists; + } /** * If not exists. @@ -403,8 +265,9 @@ public Builder ifNotExists(boolean ifNotExists) { * @param partCols the partition cols * @return the builder */ + @Deprecated // @deprecated in favour of {@link HCatTable.#partCols(List)}. To be removed in Hive 0.16. public Builder partCols(List partCols) { - this.partCols = partCols; + this.hcatTable.partCols(partCols); return this; } @@ -415,9 +278,10 @@ public Builder partCols(List partCols) { * @param bucketCols the bucket cols * @return the builder */ + @Deprecated // @deprecated in favour of {@link HCatTable.#bucketCols(List) and HCatTable.#numBuckets(int)}. + // To be removed in Hive 0.16. public Builder bucketCols(List bucketCols, int buckets) { - this.bucketCols = bucketCols; - this.numBuckets = buckets; + this.hcatTable.bucketCols(bucketCols).numBuckets(buckets); return this; } @@ -427,8 +291,9 @@ public Builder bucketCols(List bucketCols, int buckets) { * @param storageHandler the storage handler * @return the builder */ - public Builder storageHandler(String storageHandler) { - this.storageHandler = storageHandler; + @Deprecated // @deprecated in favour of {@link HCatTable.#storageHandler(String)}. To be removed in Hive 0.16. + public Builder storageHandler(String storageHandler) throws HCatException { + this.hcatTable.storageHandler(storageHandler); return this; } @@ -438,8 +303,9 @@ public Builder storageHandler(String storageHandler) { * @param location the location * @return the builder */ + @Deprecated // @deprecated in favour of {@link HCatTable.#location(String)}. To be removed in Hive 0.16. public Builder location(String location) { - this.location = location; + this.hcatTable.location(location); return this; } @@ -449,8 +315,9 @@ public Builder location(String location) { * @param comment the comment * @return the builder */ + @Deprecated // @deprecated in favour of {@link HCatTable.#comment(String)}. To be removed in Hive 0.16. public Builder comments(String comment) { - this.comment = comment; + this.hcatTable.comment(comment); return this; } @@ -460,8 +327,9 @@ public Builder comments(String comment) { * @param isExternal the is external * @return the builder */ + @Deprecated // @deprecated in favour of {@link HCatTable.#tableType(HCatTable.Type)}. To be removed in Hive 0.16. public Builder isTableExternal(boolean isExternal) { - this.isExternal = isExternal; + this.hcatTable.tableType(isExternal? HCatTable.Type.EXTERNAL_TABLE : HCatTable.Type.MANAGED_TABLE); return this; } @@ -471,8 +339,9 @@ public Builder isTableExternal(boolean isExternal) { * @param sortCols the sort cols * @return the builder */ + @Deprecated // @deprecated in favour of {@link HCatTable.#sortCols(ArrayList)}. To be removed in Hive 0.16. public Builder sortCols(ArrayList sortCols) { - this.sortCols = sortCols; + this.hcatTable.sortCols(sortCols); return this; } @@ -482,8 +351,10 @@ public Builder sortCols(ArrayList sortCols) { * @param tblProps the tbl props * @return the builder */ + @Deprecated // @deprecated in favour of {@link HCatTable.#sortCols(Map)}. + // To be removed in Hive 0.16. public Builder tblProps(Map tblProps) { - this.tblProps = tblProps; + this.hcatTable.tblProps(tblProps); return this; } @@ -493,54 +364,60 @@ public Builder tblProps(Map tblProps) { * @param format the format * @return the builder */ + @Deprecated // @deprecated in favour of {@link HCatTable.#fileFormat(String)}. To be removed in Hive 0.16. public Builder fileFormat(String format) { - this.fileFormat = format; + this.hcatTable.fileFormat(format); return this; } /** * See row_format element of CREATE_TABLE DDL for Hive. */ + @Deprecated // @deprecated in favour of {@link HCatTable.#fieldsTerminatedBy()}. To be removed in Hive 0.16. public Builder fieldsTerminatedBy(char delimiter) { return serdeParam(serdeConstants.FIELD_DELIM, Character.toString(delimiter)); } /** * See row_format element of CREATE_TABLE DDL for Hive. */ + @Deprecated // @deprecated in favour of {@link HCatTable.#escapeChar()}. public Builder escapeChar(char escapeChar) { return serdeParam(serdeConstants.ESCAPE_CHAR, Character.toString(escapeChar)); } /** * See row_format element of CREATE_TABLE DDL for Hive. */ + @Deprecated // @deprecated in favour of {@link HCatTable.#collectionItemsTerminatedBy()}. To be removed in Hive 0.16. public Builder collectionItemsTerminatedBy(char delimiter) { return serdeParam(serdeConstants.COLLECTION_DELIM, Character.toString(delimiter)); } /** * See row_format element of CREATE_TABLE DDL for Hive. */ + @Deprecated // @deprecated in favour of {@link HCatTable.#mapKeysTerminatedBy()}. To be removed in Hive 0.16. public Builder mapKeysTerminatedBy(char delimiter) { return serdeParam(serdeConstants.MAPKEY_DELIM, Character.toString(delimiter)); } /** * See row_format element of CREATE_TABLE DDL for Hive. */ + @Deprecated // @deprecated in favour of {@link HCatTable.#linesTerminatedBy()}. To be removed in Hive 0.16. public Builder linesTerminatedBy(char delimiter) { return serdeParam(serdeConstants.LINE_DELIM, Character.toString(delimiter)); } /** * See row_format element of CREATE_TABLE DDL for Hive. */ + @Deprecated // @deprecated in favour of {@link HCatTable.#nullDefinedAs()}. To be removed in Hive 0.16. public Builder nullDefinedAs(char nullChar) { return serdeParam(serdeConstants.SERIALIZATION_NULL_FORMAT, Character.toString(nullChar)); } /** * used for setting arbitrary SerDe parameter */ + @Deprecated // @deprecated in favour of {@link HCatTable.#serdeParam(Map)}. + // To be removed in Hive 0.16. public Builder serdeParam(String paramName, String value) { - if(serdeParams == null) { - serdeParams = new HashMap(); - } - serdeParams.put(paramName, value); + hcatTable.serdeParam(paramName, value); return this; } /** @@ -550,52 +427,9 @@ public Builder serdeParam(String paramName, String value) { * @throws HCatException */ public HCatCreateTableDesc build() throws HCatException { - if (this.dbName == null) { - LOG.info("Database name found null. Setting db to :" - + MetaStoreUtils.DEFAULT_DATABASE_NAME); - this.dbName = MetaStoreUtils.DEFAULT_DATABASE_NAME; - } - HCatCreateTableDesc desc = new HCatCreateTableDesc(this.dbName, - this.tableName, this.cols); - desc.ifNotExists = this.ifNotExists; - desc.isExternal = this.isExternal; - desc.comment = this.comment; - desc.partCols = this.partCols; - desc.bucketCols = this.bucketCols; - desc.numBuckets = this.numBuckets; - desc.location = this.location; - desc.tblProps = this.tblProps; - desc.sortCols = this.sortCols; - desc.serde = null; - if (!StringUtils.isEmpty(fileFormat)) { - desc.fileFormat = fileFormat; - if ("SequenceFile".equalsIgnoreCase(fileFormat)) { - desc.inputformat = SequenceFileInputFormat.class.getName(); - desc.outputformat = HiveSequenceFileOutputFormat.class - .getName(); - } else if ("RCFile".equalsIgnoreCase(fileFormat)) { - desc.inputformat = RCFileInputFormat.class.getName(); - desc.outputformat = RCFileOutputFormat.class.getName(); - desc.serde = ColumnarSerDe.class.getName(); - } else if ("orcfile".equalsIgnoreCase(fileFormat)) { - desc.inputformat = OrcInputFormat.class.getName(); - desc.outputformat = OrcOutputFormat.class.getName(); - desc.serde = OrcSerde.class.getName(); - } - desc.storageHandler = StringUtils.EMPTY; - } else if (!StringUtils.isEmpty(storageHandler)) { - desc.storageHandler = storageHandler; - } else { - desc.fileFormat = "TextFile"; - LOG.info("Using text file format for the table."); - desc.inputformat = TextInputFormat.class.getName(); - LOG.info("Table input format:" + desc.inputformat); - desc.outputformat = HiveIgnoreKeyTextOutputFormat.class - .getName(); - LOG.info("Table output format:" + desc.outputformat); - } - desc.serdeParams = this.serdeParams; - return desc; + return new HCatCreateTableDesc(this.hcatTable, this.ifNotExists); } - } -} + + } // class Builder; + +} // class HCatAddPartitionDesc; diff --git a/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatPartition.java b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatPartition.java index ee1b6bf..ea7c54c 100644 --- a/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatPartition.java +++ b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatPartition.java @@ -19,45 +19,136 @@ package org.apache.hive.hcatalog.api; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hive.hcatalog.common.HCatException; import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; import org.apache.hive.hcatalog.data.schema.HCatSchemaUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * The HCatPartition is a wrapper around org.apache.hadoop.hive.metastore.api.Partition. */ public class HCatPartition { + private static final Logger LOG = LoggerFactory.getLogger(HCatPartition.class); + + private HCatTable hcatTable; private String tableName; - private String dbName; + private String dbName = MetaStoreUtils.DEFAULT_DATABASE_NAME; private List values; - private List tableCols; private int createTime; private int lastAccessTime; private StorageDescriptor sd; private Map parameters; - HCatPartition(Partition partition) throws HCatException { + // For use from within HCatClient.getPartitions(). + HCatPartition(HCatTable hcatTable, Partition partition) throws HCatException { + this.hcatTable = hcatTable; this.tableName = partition.getTableName(); this.dbName = partition.getDbName(); this.createTime = partition.getCreateTime(); this.lastAccessTime = partition.getLastAccessTime(); this.parameters = partition.getParameters(); this.values = partition.getValues(); + if (hcatTable != null && partition.getValuesSize() != hcatTable.getPartCols().size()) { + throw new HCatException("Mismatched number of partition columns between table:" + hcatTable.getDbName() + "." + hcatTable.getTableName() + + " and partition " + partition.getValues()); + } + this.sd = partition.getSd(); - this.tableCols = new ArrayList(); - for (FieldSchema fs : this.sd.getCols()) { - this.tableCols.add(HCatSchemaUtils.getHCatFieldSchema(fs)); + } + + // For constructing HCatPartitions afresh, as an argument to HCatClient.addPartitions(). + public HCatPartition(HCatTable hcatTable, Map partitionKeyValues, String location) throws HCatException { + this.hcatTable = hcatTable; + this.tableName = hcatTable.getTableName(); + this.dbName = hcatTable.getDbName(); + this.sd = new StorageDescriptor(hcatTable.getSd()); + this.sd.setLocation(location); + this.createTime = (int)(System.currentTimeMillis()/1000); + this.lastAccessTime = -1; + this.values = new ArrayList(hcatTable.getPartCols().size()); + for (HCatFieldSchema partField : hcatTable.getPartCols()) { + if (!partitionKeyValues.containsKey(partField.getName())) { + throw new HCatException("Missing value for partition-key \'" + partField.getName() + + "\' in table: " + hcatTable.getDbName() + "." + hcatTable.getTableName()); + } + else { + values.add(partitionKeyValues.get(partField.getName())); + } } } + // For replicating an HCatPartition definition. + public HCatPartition(HCatPartition rhs, Map partitionKeyValues, String location) throws HCatException { + this.hcatTable = rhs.hcatTable; + this.tableName = rhs.tableName; + this.dbName = rhs.dbName; + this.sd = new StorageDescriptor(rhs.sd); + this.sd.setLocation(location); + + this.createTime = (int) (System.currentTimeMillis() / 1000); + this.lastAccessTime = -1; + this.values = new ArrayList(hcatTable.getPartCols().size()); + for (HCatFieldSchema partField : hcatTable.getPartCols()) { + if (!partitionKeyValues.containsKey(partField.getName())) { + throw new HCatException("Missing value for partition-key \'" + partField.getName() + + "\' in table: " + hcatTable.getDbName() + "." + hcatTable.getTableName()); + } else { + values.add(partitionKeyValues.get(partField.getName())); + } + } + } + + // For use from HCatClient.addPartitions(), to construct from user-input. + Partition toHivePartition() throws HCatException { + Partition hivePtn = new Partition(); + hivePtn.setDbName(dbName); + hivePtn.setTableName(tableName); + hivePtn.setValues(values); + + hivePtn.setParameters(parameters); + if (sd.getLocation() == null) { + LOG.warn("Partition location is not set! Attempting to construct default partition location."); + try { + String partName = Warehouse.makePartName(HCatSchemaUtils.getFieldSchemas(hcatTable.getPartCols()), values); + sd.setLocation(new Path(hcatTable.getSd().getLocation(), partName).toString()); + } + catch(MetaException exception) { + throw new HCatException("Could not construct default partition-path for " + + hcatTable.getDbName() + "." + hcatTable.getTableName() + "[" + values + "]"); + } + } + hivePtn.setSd(sd); + + hivePtn.setCreateTime((int) (System.currentTimeMillis() / 1000)); + hivePtn.setLastAccessTimeIsSet(false); + return hivePtn; + } + + public HCatTable hcatTable() { + return hcatTable; + } + + public HCatPartition hcatTable(HCatTable hcatTable) { + this.hcatTable = hcatTable; + this.tableName = hcatTable.getTableName(); + this.dbName = hcatTable.getDbName(); + return this; + } + /** * Gets the table name. * @@ -81,8 +172,12 @@ public String getDatabaseName() { * * @return the columns */ - public List getColumns() { - return this.tableCols; + public List getColumns() throws HCatException { + ArrayList columns = new ArrayList(sd.getColsSize()); + for (FieldSchema fieldSchema : sd.getCols()) { + columns.add(HCatSchemaUtils.getHCatFieldSchema(fieldSchema)); + } + return columns; } /** @@ -124,6 +219,14 @@ public String getLocation() { } /** + * Setter for partition directory location. + */ + public HCatPartition location(String location) { + this.sd.setLocation(location); + return this; + } + + /** * Gets the serde. * * @return the serde @@ -132,6 +235,14 @@ public String getSerDe() { return this.sd.getSerdeInfo().getSerializationLib(); } + /** + * Getter for SerDe parameters. + * @return The SerDe parameters. + */ + public Map getSerdeParams() { + return this.sd.getSerdeInfo().getParameters(); + } + public Map getParameters() { return this.parameters; } @@ -164,6 +275,34 @@ public int getCreateTime() { } /** + * Getter for partition-spec map. + */ + public LinkedHashMap getPartitionKeyValMap() { + LinkedHashMap map = new LinkedHashMap(hcatTable.getPartCols().size()); + for (int i=0; i partitionKeyValues) throws HCatException { + for (HCatFieldSchema partField : hcatTable.getPartCols()) { + if (!partitionKeyValues.containsKey(partField.getName())) { + throw new HCatException("Missing value for partition-key \'" + partField.getName() + + "\' in table: " + hcatTable.getDbName() + "." + hcatTable.getTableName()); + } + else { + values.add(partitionKeyValues.get(partField.getName())); + // Keep partKeyValMap in synch as well. + } + } + return this; + } + + /** * Gets the bucket columns. * * @return the bucket columns @@ -192,13 +331,14 @@ public int getNumBuckets() { @Override public String toString() { - return "HCatPartition [" - + (tableName != null ? "tableName=" + tableName + ", " : "tableName=null") - + (dbName != null ? "dbName=" + dbName + ", " : "dbName=null") - + (values != null ? "values=" + values + ", " : "values=null") - + "createTime=" + createTime + ", lastAccessTime=" - + lastAccessTime + ", " + (sd != null ? "sd=" + sd + ", " : "sd=null") - + (parameters != null ? "parameters=" + parameters : "parameters=null") + "]"; + return "HCatPartition [ " + + "tableName=" + tableName + "," + + "dbName=" + dbName + "," + + "values=" + values + "," + + "createTime=" + createTime + "," + + "lastAccessTime=" + lastAccessTime + "," + + "sd=" + sd + "," + + "parameters=" + parameters + "]"; } } diff --git a/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatTable.java b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatTable.java index be254c3..6c1b5df 100644 --- a/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatTable.java +++ b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatTable.java @@ -19,62 +19,200 @@ package org.apache.hive.hcatalog.api; import java.util.ArrayList; +import java.util.EnumSet; +import java.util.HashMap; import java.util.List; import java.util.Map; +import com.google.common.collect.Maps; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Order; +import org.apache.hadoop.hive.metastore.api.SerDeInfo; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat; +import org.apache.hadoop.hive.ql.io.RCFileInputFormat; +import org.apache.hadoop.hive.ql.io.RCFileOutputFormat; +import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; +import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat; +import org.apache.hadoop.hive.ql.io.orc.OrcSerde; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; +import org.apache.hadoop.hive.ql.metadata.HiveUtils; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe; +import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; +import org.apache.hadoop.mapred.SequenceFileInputFormat; import org.apache.hive.hcatalog.common.HCatException; import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; import org.apache.hive.hcatalog.data.schema.HCatSchemaUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * The HCatTable is a wrapper around org.apache.hadoop.hive.metastore.api.Table. */ public class HCatTable { + private static final Logger LOG = LoggerFactory.getLogger(HCatTable.class); + public static enum Type { + MANAGED_TABLE, + EXTERNAL_TABLE, + VIRTUAL_VIEW, + INDEX_TABLE + } + + /** + * Attributes that can be compared between HCatTables. + */ + public static enum TableAttribute { + COLUMNS, + PARTITION_COLUMNS, + INPUT_FORMAT, + OUTPUT_FORMAT, + SERDE, + SERDE_PROPERTIES, + STORAGE_HANDLER, + LOCATION, + TABLE_PROPERTIES, + STATS // TODO: Handle replication of changes to Table-STATS. + } + + /** + * The default set of attributes that can be diffed between HCatTables. + */ + public static final EnumSet DEFAULT_COMPARISON_ATTRIBUTES + = EnumSet.of(TableAttribute.COLUMNS, + TableAttribute.INPUT_FORMAT, + TableAttribute.OUTPUT_FORMAT, + TableAttribute.SERDE, + TableAttribute.SERDE_PROPERTIES, + TableAttribute.STORAGE_HANDLER, + TableAttribute.TABLE_PROPERTIES); + + /** + * 2 HCatTables are considered equivalent if {@code lhs.diff(rhs).equals(NO_DIFF) == true; } + */ + public static final EnumSet NO_DIFF = EnumSet.noneOf(TableAttribute.class); + + public static final String DEFAULT_SERDE_CLASS = org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName(); + public static final String DEFAULT_INPUT_FORMAT_CLASS = org.apache.hadoop.mapred.TextInputFormat.class.getName(); + public static final String DEFAULT_OUTPUT_FORMAT_CLASS = org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat.class.getName(); + + private String dbName = MetaStoreUtils.DEFAULT_DATABASE_NAME; private String tableName; - private String tabletype; - private List cols; - private List partCols; - private List bucketCols; - private List sortCols; - private int numBuckets; - private String inputFileFormat; - private String outputFileFormat; - private String storageHandler; - private Map tblProps; - private String dbName; - private String serde; - private String location; - private Map serdeParams; + private HiveConf conf; + private String tableType; + private boolean isExternal; + private List cols = new ArrayList(); + private List partCols = new ArrayList(); + private StorageDescriptor sd; + private String fileFormat; + private Map tblProps = new HashMap(); + private String comment = ""; + private String owner; + + public HCatTable(String dbName, String tableName) { + this.dbName = StringUtils.isBlank(dbName)? MetaStoreUtils.DEFAULT_DATABASE_NAME : dbName; + this.tableName = tableName; + this.sd = new StorageDescriptor(); + this.sd.setInputFormat(DEFAULT_INPUT_FORMAT_CLASS); + this.sd.setOutputFormat(DEFAULT_OUTPUT_FORMAT_CLASS); + this.sd.setSerdeInfo(new SerDeInfo()); + this.sd.getSerdeInfo().setSerializationLib(DEFAULT_SERDE_CLASS); + this.sd.getSerdeInfo().setParameters(new HashMap()); + this.sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1"); // Default serialization format. + } HCatTable(Table hiveTable) throws HCatException { - this.tableName = hiveTable.getTableName(); - this.dbName = hiveTable.getDbName(); - this.tabletype = hiveTable.getTableType(); - cols = new ArrayList(); - for (FieldSchema colFS : hiveTable.getSd().getCols()) { + tableName = hiveTable.getTableName(); + dbName = hiveTable.getDbName(); + tableType = hiveTable.getTableType(); + isExternal = hiveTable.getTableType().equals(TableType.EXTERNAL_TABLE.toString()); + sd = hiveTable.getSd(); + for (FieldSchema colFS : sd.getCols()) { cols.add(HCatSchemaUtils.getHCatFieldSchema(colFS)); } partCols = new ArrayList(); for (FieldSchema colFS : hiveTable.getPartitionKeys()) { partCols.add(HCatSchemaUtils.getHCatFieldSchema(colFS)); } - bucketCols = hiveTable.getSd().getBucketCols(); - sortCols = hiveTable.getSd().getSortCols(); - numBuckets = hiveTable.getSd().getNumBuckets(); - inputFileFormat = hiveTable.getSd().getInputFormat(); - outputFileFormat = hiveTable.getSd().getOutputFormat(); - storageHandler = hiveTable - .getSd() - .getParameters() - .get(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE); - tblProps = hiveTable.getParameters(); - serde = hiveTable.getSd().getSerdeInfo().getSerializationLib(); - location = hiveTable.getSd().getLocation(); - serdeParams = hiveTable.getSd().getSerdeInfo().getParameters(); + if (hiveTable.getParameters() != null) { + tblProps.putAll(hiveTable.getParameters()); + } + + if (StringUtils.isNotBlank(tblProps.get("comment"))) { + comment = tblProps.get("comment"); + } + + owner = hiveTable.getOwner(); + } + + Table toHiveTable() throws HCatException { + Table newTable = new Table(); + newTable.setDbName(dbName); + newTable.setTableName(tableName); + if (tblProps != null) { + newTable.setParameters(tblProps); + } + + if (isExternal) { + newTable.putToParameters("EXTERNAL", "TRUE"); + newTable.setTableType(TableType.EXTERNAL_TABLE.toString()); + } else { + newTable.setTableType(TableType.MANAGED_TABLE.toString()); + } + + if (this.comment != null) { + newTable.putToParameters("comment", comment); + } + + newTable.setSd(sd); + if (partCols != null) { + ArrayList hivePtnCols = new ArrayList(); + for (HCatFieldSchema fs : partCols) { + hivePtnCols.add(HCatSchemaUtils.getFieldSchema(fs)); + } + newTable.setPartitionKeys(hivePtnCols); + } + + newTable.setCreateTime((int) (System.currentTimeMillis() / 1000)); + newTable.setLastAccessTimeIsSet(false); + try { + // TODO: Verify that this works for systems using UGI.doAs() (e.g. Oozie). + newTable.setOwner(owner == null? getConf().getUser() : owner); + } + catch (Exception exception) { + throw new HCatException("Unable to determine owner of table (" + dbName + "." + tableName + + ") from HiveConf."); + } + return newTable; + } + + void setConf(Configuration conf) { + if (conf instanceof HiveConf) { + this.conf = (HiveConf)conf; + } + else { + this.conf = new HiveConf(conf, getClass()); + } + } + + HiveConf getConf() { + if (conf == null) { + LOG.warn("Conf hasn't been set yet. Using defaults."); + conf = new HiveConf(); + } + return conf; + } + + StorageDescriptor getSd() { + return sd; } /** @@ -87,6 +225,14 @@ public String getTableName() { } /** + * Setter for TableName. + */ + public HCatTable tableName(String tableName) { + this.tableName = tableName; + return this; + } + + /** * Gets the db name. * * @return the db name @@ -96,6 +242,14 @@ public String getDbName() { } /** + * Setter for db-name. + */ + public HCatTable dbName(String dbName) { + this.dbName = dbName; + return this; + } + + /** * Gets the columns. * * @return the columns @@ -105,6 +259,18 @@ public String getDbName() { } /** + * Setter for Column schemas. + */ + public HCatTable cols(List cols) { + if (!this.cols.equals(cols)) { + this.cols.clear(); + this.cols.addAll(cols); + this.sd.setCols(HCatSchemaUtils.getFieldSchemas(cols)); + } + return this; + } + + /** * Gets the part columns. * * @return the part columns @@ -114,12 +280,40 @@ public String getDbName() { } /** + * Setter for list of partition columns. + */ + public HCatTable partCols(List partCols) { + this.partCols = partCols; + return this; + } + + /** + * Setter for individual partition columns. + */ + public HCatTable partCol(HCatFieldSchema partCol) { + if (this.partCols == null) { + this.partCols = new ArrayList(); + } + + this.partCols.add(partCol); + return this; + } + + /** * Gets the bucket columns. * * @return the bucket columns */ public List getBucketCols() { - return bucketCols; + return this.sd.getBucketCols(); + } + + /** + * Setter for list of bucket columns. + */ + public HCatTable bucketCols(List bucketCols) { + this.sd.setBucketCols(bucketCols); + return this; } /** @@ -128,7 +322,15 @@ public String getDbName() { * @return the sort columns */ public List getSortCols() { - return sortCols; + return this.sd.getSortCols(); + } + + /** + * Setter for Sort-cols. + */ + public HCatTable sortCols(List sortCols) { + this.sd.setSortCols(sortCols); + return this; } /** @@ -137,7 +339,15 @@ public String getDbName() { * @return the number of buckets */ public int getNumBuckets() { - return numBuckets; + return this.sd.getNumBuckets(); + } + + /** + * Setter for number of buckets. + */ + public HCatTable numBuckets(int numBuckets) { + this.sd.setNumBuckets(numBuckets); + return this; } /** @@ -146,7 +356,30 @@ public int getNumBuckets() { * @return the storage handler */ public String getStorageHandler() { - return storageHandler; + return this.tblProps.get(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE); + } + + /** + * Setter for StorageHandler class. + */ + public HCatTable storageHandler(String storageHandler) throws HCatException { + this.tblProps.put( + org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE, + storageHandler); + LOG.warn("HiveStorageHandlers can't be instantiated on the client-side. " + + "Replication of StorageHandler-based tables is not supported at this time. " + + "Attempting to derive Input/OutputFormat settings from StorageHandler, on best effort: "); + try { + HiveStorageHandler sh = HiveUtils.getStorageHandler(getConf(), storageHandler); + this.sd.setInputFormat(sh.getInputFormatClass().getName()); + this.sd.setOutputFormat(sh.getOutputFormatClass().getName()); + this.sd.getSerdeInfo().setSerializationLib(sh.getSerDeClass().getName()); + } catch (HiveException e) { + LOG.warn("Could not derive Input/OutputFormat and SerDe settings from storageHandler. " + + "These values need to be set explicitly.", e); + } + + return this; } /** @@ -159,21 +392,83 @@ public String getStorageHandler() { } /** - * Gets the tabletype. + * Setter for TableProperty map. + */ + public HCatTable tblProps(Map tblProps) { + if (!this.tblProps.equals(tblProps)) { + this.tblProps.clear(); + this.tblProps.putAll(tblProps); + } + return this; + } + + /** + * Gets the tableType. * - * @return the tabletype + * @return the tableType */ public String getTabletype() { - return tabletype; + return tableType; + } + + /** + * Setter for table-type. + */ + public HCatTable tableType(Type tableType) { + this.tableType = tableType.name(); + this.isExternal = tableType.equals(Type.EXTERNAL_TABLE); + return this; + } + + private SerDeInfo getSerDeInfo() { + if (!sd.isSetSerdeInfo()) { + sd.setSerdeInfo(new SerDeInfo()); + } + return sd.getSerdeInfo(); + } + + public HCatTable fileFormat(String fileFormat) { + this.fileFormat = fileFormat; + + if (fileFormat.equalsIgnoreCase("sequencefile")) { + inputFileFormat(SequenceFileInputFormat.class.getName()); + outputFileFormat(HiveSequenceFileOutputFormat.class.getName()); + serdeLib(LazySimpleSerDe.class.getName()); + } + else + if (fileFormat.equalsIgnoreCase("rcfile")) { + inputFileFormat(RCFileInputFormat.class.getName()); + outputFileFormat(RCFileOutputFormat.class.getName()); + serdeLib(LazyBinaryColumnarSerDe.class.getName()); + } + else + if (fileFormat.equalsIgnoreCase("orcfile")) { + inputFileFormat(OrcInputFormat.class.getName()); + outputFileFormat(OrcOutputFormat.class.getName()); + serdeLib(OrcSerde.class.getName()); + } + + return this; } + public String fileFormat() { + return fileFormat; + } /** * Gets the input file format. * * @return the input file format */ public String getInputFileFormat() { - return inputFileFormat; + return sd.getInputFormat(); + } + + /** + * Setter for InputFormat class. + */ + public HCatTable inputFileFormat(String inputFileFormat) { + sd.setInputFormat(inputFileFormat); + return this; } /** @@ -182,7 +477,15 @@ public String getInputFileFormat() { * @return the output file format */ public String getOutputFileFormat() { - return outputFileFormat; + return sd.getOutputFormat(); + } + + /** + * Setter for OutputFormat class. + */ + public HCatTable outputFileFormat(String outputFileFormat) { + this.sd.setOutputFormat(outputFileFormat); + return this; } /** @@ -191,7 +494,37 @@ public String getOutputFileFormat() { * @return the serde lib */ public String getSerdeLib() { - return serde; + return getSerDeInfo().getSerializationLib(); + } + + /** + * Setter for SerDe class name. + */ + public HCatTable serdeLib(String serde) { + getSerDeInfo().setSerializationLib(serde); + return this; + } + + public HCatTable serdeParams(Map serdeParams) { + getSerDeInfo().setParameters(serdeParams); + return this; + } + + public HCatTable serdeParam(String paramName, String value) { + SerDeInfo serdeInfo = getSerDeInfo(); + if (serdeInfo.getParameters() == null) { + serdeInfo.setParameters(new HashMap()); + } + serdeInfo.getParameters().put(paramName, value); + + return this; + } + + /** + * Returns parameters such as field delimiter,etc. + */ + public Map getSerdeParams() { + return getSerDeInfo().getParameters(); } /** @@ -200,38 +533,230 @@ public String getSerdeLib() { * @return the location */ public String getLocation() { - return location; + return sd.getLocation(); } + /** - * Returns parameters such as field delimiter,etc. + * Setter for location. */ - public Map getSerdeParams() { - return serdeParams; + public HCatTable location(String location) { + this.sd.setLocation(location); + return this; + } + + /** + * Getter for table-owner. + */ + public String owner() { + return owner; + } + + /** + * Setter for table-owner. + */ + public HCatTable owner(String owner) { + this.owner = owner; + return this; + } + + public String comment() { + return this.comment; + } + + /** + * Setter for table-level comment. + */ + public HCatTable comment(String comment) { + this.comment = comment; + return this; + } + + /** + * See row_format element of CREATE_TABLE DDL for Hive. + */ + public HCatTable fieldsTerminatedBy(char delimiter) { + return serdeParam(serdeConstants.FIELD_DELIM, Character.toString(delimiter)); + } + /** + * See row_format element of CREATE_TABLE DDL for Hive. + */ + public HCatTable escapeChar(char escapeChar) { + return serdeParam(serdeConstants.ESCAPE_CHAR, Character.toString(escapeChar)); + } + /** + * See row_format element of CREATE_TABLE DDL for Hive. + */ + public HCatTable collectionItemsTerminatedBy(char delimiter) { + return serdeParam(serdeConstants.COLLECTION_DELIM, Character.toString(delimiter)); + } + /** + * See row_format element of CREATE_TABLE DDL for Hive. + */ + public HCatTable mapKeysTerminatedBy(char delimiter) { + return serdeParam(serdeConstants.MAPKEY_DELIM, Character.toString(delimiter)); + } + /** + * See row_format element of CREATE_TABLE DDL for Hive. + */ + public HCatTable linesTerminatedBy(char delimiter) { + return serdeParam(serdeConstants.LINE_DELIM, Character.toString(delimiter)); + } + /** + * See row_format element of CREATE_TABLE DDL for Hive. + */ + public HCatTable nullDefinedAs(char nullChar) { + return serdeParam(serdeConstants.SERIALIZATION_NULL_FORMAT, Character.toString(nullChar)); } @Override public String toString() { - return "HCatTable [" - + (tableName != null ? "tableName=" + tableName + ", " : "tableName=null") - + (dbName != null ? "dbName=" + dbName + ", " : "dbName=null") - + (tabletype != null ? "tabletype=" + tabletype + ", " : "tabletype=null") - + (cols != null ? "cols=" + cols + ", " : "cols=null") - + (partCols != null ? "partCols=" + partCols + ", " : "partCols==null") - + (bucketCols != null ? "bucketCols=" + bucketCols + ", " : "bucketCols=null") - + (sortCols != null ? "sortCols=" + sortCols + ", " : "sortCols=null") - + "numBuckets=" - + numBuckets - + ", " - + (inputFileFormat != null ? "inputFileFormat=" - + inputFileFormat + ", " : "inputFileFormat=null") - + (outputFileFormat != null ? "outputFileFormat=" - + outputFileFormat + ", " : "outputFileFormat=null") - + (storageHandler != null ? "storageHandler=" + storageHandler - + ", " : "storageHandler=null") - + (tblProps != null ? "tblProps=" + tblProps + ", " : "tblProps=null") - + (serde != null ? "serde=" + serde + ", " : "serde=") - + (location != null ? "location=" + location : "location=") - + ",serdeParams=" + (serdeParams == null ? "null" : serdeParams) - + "]"; + return "HCatTable [ " + + "tableName=" + tableName + ", " + + "dbName=" + dbName + ", " + + "tableType=" + tableType + ", " + + "cols=" + cols + ", " + + "partCols=" + partCols + ", " + + "bucketCols=" + getBucketCols() + ", " + + "numBuckets=" + getNumBuckets() + ", " + + "sortCols=" + getSortCols() + ", " + + "inputFormat=" + getInputFileFormat() + ", " + + "outputFormat=" + getOutputFileFormat() + ", " + + "storageHandler=" + getStorageHandler() + ", " + + "serde=" + getSerdeLib() + ", " + + "tblProps=" + getTblProps() + ", " + + "location=" + getLocation() + ", " + + "owner=" + owner() + " ]"; + + } + + /** + * Method to compare the attributes of 2 HCatTable instances. + * @param rhs The other table being compared against. Can't be null. + * @param attributesToCheck The list of TableAttributes being compared. + * @return {@code EnumSet} containing all the attribute that differ between {@code this} and rhs. + * Subset of {@code attributesToCheck}. + */ + public EnumSet diff(HCatTable rhs, EnumSet attributesToCheck) { + EnumSet theDiff = EnumSet.noneOf(TableAttribute.class); + + for (TableAttribute attribute : attributesToCheck) { + + if (attribute.equals(TableAttribute.COLUMNS)) { + if (!rhs.getCols().containsAll(getCols()) || + !getCols().containsAll(rhs.getCols())) { + theDiff.add(TableAttribute.COLUMNS); + } + } + + if (attribute.equals(TableAttribute.INPUT_FORMAT)) { + if ((getInputFileFormat() == null && rhs.getInputFileFormat() != null) + || (getInputFileFormat() != null && (rhs.getInputFileFormat() == null || !rhs.getInputFileFormat().equals(getInputFileFormat())))) { + theDiff.add(TableAttribute.INPUT_FORMAT); + } + } + + if (attribute.equals(TableAttribute.OUTPUT_FORMAT)) { + if ((getOutputFileFormat() == null && rhs.getOutputFileFormat() != null) + || (getOutputFileFormat() != null && (rhs.getOutputFileFormat() == null || !rhs.getOutputFileFormat().equals(getOutputFileFormat())))) { + theDiff.add(TableAttribute.OUTPUT_FORMAT); + } + } + + if (attribute.equals(TableAttribute.STORAGE_HANDLER)) { + if ((getStorageHandler() == null && rhs.getStorageHandler() != null) + || (getStorageHandler() != null && (rhs.getStorageHandler() == null || !rhs.getStorageHandler().equals(getStorageHandler())))) { + theDiff.add(TableAttribute.STORAGE_HANDLER); + } + } + + if (attribute.equals(TableAttribute.SERDE)) { + if ((getSerdeLib() == null && rhs.getSerdeLib() != null) + || (getSerdeLib() != null && (rhs.getSerdeLib() == null || !rhs.getSerdeLib().equals(getSerdeLib())))) { + theDiff.add(TableAttribute.SERDE); + } + } + + if (attribute.equals(TableAttribute.SERDE_PROPERTIES)) { + if (!equivalent(sd.getSerdeInfo().getParameters(), rhs.sd.getSerdeInfo().getParameters())) { + theDiff.add(TableAttribute.SERDE_PROPERTIES); + } + } + + if (attribute.equals(TableAttribute.TABLE_PROPERTIES)) { + if (!equivalent(tblProps, rhs.tblProps)) { + theDiff.add(TableAttribute.TABLE_PROPERTIES); + } + } + + } + + return theDiff; + } + + /** + * Helper method to compare 2 Map instances, for equivalence. + * @param lhs First map to be compared. + * @param rhs Second map to be compared. + * @return true, if the 2 Maps contain the same entries. + */ + private static boolean equivalent(Map lhs, Map rhs) { + return lhs.size() == rhs.size() && Maps.difference(lhs, rhs).areEqual(); + } + + /** + * Method to compare the attributes of 2 HCatTable instances. + * Only the {@code DEFAULT_COMPARISON_ATTRIBUTES} are compared. + * @param rhs The other table being compared against. Can't be null. + * @return {@code EnumSet} containing all the attribute that differ between {@code this} and rhs. + * Subset of {@code DEFAULT_COMPARISON_ATTRIBUTES}. + */ + public EnumSet diff (HCatTable rhs) { + return diff(rhs, DEFAULT_COMPARISON_ATTRIBUTES); + } + + /** + * Method to "adopt" the specified attributes from rhs into this HCatTable object. + * @param rhs The "source" table from which attributes are to be copied from. + * @param attributes The set of attributes to be copied from rhs. Usually the result of {@code this.diff(rhs)}. + * @return This HCatTable + * @throws HCatException + */ + public HCatTable resolve(HCatTable rhs, EnumSet attributes) throws HCatException { + + if (rhs == this) + return this; + + for (TableAttribute attribute : attributes) { + + if (attribute.equals(TableAttribute.COLUMNS)) { + cols(rhs.cols); + } + + if (attribute.equals(TableAttribute.INPUT_FORMAT)) { + inputFileFormat(rhs.getInputFileFormat()); + } + + if (attribute.equals(TableAttribute.OUTPUT_FORMAT)) { + outputFileFormat(rhs.getOutputFileFormat()); + } + + if (attribute.equals(TableAttribute.SERDE)) { + serdeLib(rhs.getSerdeLib()); + } + + if (attribute.equals(TableAttribute.SERDE_PROPERTIES)) { + serdeParams(rhs.getSerdeParams()); + } + + if (attribute.equals(TableAttribute.STORAGE_HANDLER)) { + storageHandler(rhs.getStorageHandler()); + } + + if (attribute.equals(TableAttribute.TABLE_PROPERTIES)) { + tblProps(rhs.tblProps); + } + } + + return this; } } diff --git a/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/MetadataJSONSerializer.java b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/MetadataJSONSerializer.java new file mode 100644 index 0000000..30ac00f --- /dev/null +++ b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/MetadataJSONSerializer.java @@ -0,0 +1,71 @@ +package org.apache.hive.hcatalog.api; + +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hive.hcatalog.common.HCatException; +import org.apache.thrift.TDeserializer; +import org.apache.thrift.TException; +import org.apache.thrift.TSerializer; +import org.apache.thrift.protocol.TJSONProtocol; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * MetadataSerializer implementation, that serializes HCat API elements into JSON. + */ +class MetadataJSONSerializer extends MetadataSerializer { + + private static final Logger LOG = LoggerFactory.getLogger(MetadataJSONSerializer.class); + + MetadataJSONSerializer() throws HCatException {} + + @Override + public String serializeTable(HCatTable hcatTable) throws HCatException { + try { + return new TSerializer(new TJSONProtocol.Factory()) + .toString(hcatTable.toHiveTable(), "UTF-8"); + } + catch (TException exception) { + throw new HCatException("Could not serialize HCatTable: " + hcatTable, exception); + } + } + + @Override + public HCatTable deserializeTable(String hcatTableStringRep) throws HCatException { + try { + Table table = new Table(); + new TDeserializer(new TJSONProtocol.Factory()).deserialize(table, hcatTableStringRep, "UTF-8"); + return new HCatTable(table); + } + catch(TException exception) { + if (LOG.isDebugEnabled()) + LOG.debug("Could not de-serialize from: " + hcatTableStringRep); + throw new HCatException("Could not de-serialize HCatTable.", exception); + } + } + + @Override + public String serializePartition(HCatPartition hcatPartition) throws HCatException { + try { + return new TSerializer(new TJSONProtocol.Factory()) + .toString(hcatPartition.toHivePartition(), "UTF-8"); + } + catch (TException exception) { + throw new HCatException("Could not serialize HCatPartition: " + hcatPartition, exception); + } + } + + @Override + public HCatPartition deserializePartition(String hcatPartitionStringRep) throws HCatException { + try { + Partition partition = new Partition(); + new TDeserializer(new TJSONProtocol.Factory()).deserialize(partition, hcatPartitionStringRep, "UTF-8"); + return new HCatPartition(null, partition); + } + catch(TException exception) { + if (LOG.isDebugEnabled()) + LOG.debug("Could not de-serialize partition from: " + hcatPartitionStringRep); + throw new HCatException("Could not de-serialize HCatPartition.", exception); + } + } +} diff --git a/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/MetadataSerializer.java b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/MetadataSerializer.java new file mode 100644 index 0000000..dd5da99 --- /dev/null +++ b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/MetadataSerializer.java @@ -0,0 +1,54 @@ +package org.apache.hive.hcatalog.api; + +import org.apache.hive.hcatalog.common.HCatException; + +/** + * Interface to serialize HCat API elements. + */ +abstract class MetadataSerializer { + + // Prevent construction outside the get() method. + protected MetadataSerializer() {} + + /** + * Static getter method for the appropriate MetadataSerializer implementation. + * @return MetadataSerializer sub-class. + * @throws HCatException On failure to construct a concrete MetadataSerializer. + */ + public static MetadataSerializer get() throws HCatException { + return new MetadataJSONSerializer(); + } + + /** + * Serializer for HCatTable instances. + * @param hcatTable The HCatTable operand, to be serialized. + * @return Serialized (i.e. String-ified) HCatTable. + * @throws HCatException On failure to serialize. + */ + public abstract String serializeTable(HCatTable hcatTable) throws HCatException ; + + /** + * Deserializer for HCatTable string-representations. + * @param hcatTableStringRep Serialized HCatTable String (gotten from serializeTable()). + * @return Deserialized HCatTable instance. + * @throws HCatException On failure to deserialize (e.g. incompatible serialization format, etc.) + */ + public abstract HCatTable deserializeTable(String hcatTableStringRep) throws HCatException; + + /** + * Serializer for HCatPartition instances. + * @param hcatPartition The HCatPartition operand, to be serialized. + * @return Serialized (i.e. String-ified) HCatPartition. + * @throws HCatException On failure to serialize. + */ + public abstract String serializePartition(HCatPartition hcatPartition) throws HCatException; + + /** + * Deserializer for HCatPartition string-representations. + * @param hcatPartitionStringRep Serialized HCatPartition String (gotten from serializePartition()). + * @return Deserialized HCatPartition instance. + * @throws HCatException On failure to deserialize (e.g. incompatible serialization format, etc.) + */ + public abstract HCatPartition deserializePartition(String hcatPartitionStringRep) throws HCatException; + +} diff --git a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java index aea3f31..f9f7b04 100644 --- a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java +++ b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java @@ -21,6 +21,7 @@ import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; +import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -38,7 +39,7 @@ import org.apache.hadoop.hive.ql.io.orc.OrcSerde; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; +import org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe; import org.apache.hadoop.mapred.TextInputFormat; import org.apache.hive.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; import org.apache.hive.hcatalog.common.HCatConstants; @@ -65,19 +66,37 @@ private static final Logger LOG = LoggerFactory.getLogger(TestHCatClient.class); private static final String msPort = "20101"; private static HiveConf hcatConf; + private static boolean isReplicationTargetHCatRunning = false; + private static final String replicationTargetHCatPort = "20102"; + private static HiveConf replicationTargetHCatConf; private static SecurityManager securityManager; private static class RunMS implements Runnable { + private final String msPort; + private List args = new ArrayList(); + + public RunMS(String msPort) { + this.msPort = msPort; + this.args.add("-v"); + this.args.add("-p"); + this.args.add(this.msPort); + } + + public RunMS arg(String arg) { + this.args.add(arg); + return this; + } + @Override public void run() { try { - HiveMetaStore.main(new String[]{"-v", "-p", msPort}); + HiveMetaStore.main(args.toArray(new String[args.size()])); } catch (Throwable t) { LOG.error("Exiting. Got exception from metastore: ", t); } } - } + } // class RunMS; @AfterClass public static void tearDown() throws Exception { @@ -88,9 +107,9 @@ public static void tearDown() throws Exception { @BeforeClass public static void startMetaStoreServer() throws Exception { - Thread t = new Thread(new RunMS()); + Thread t = new Thread(new RunMS(msPort)); t.start(); - Thread.sleep(40000); + Thread.sleep(10000); securityManager = System.getSecurityManager(); System.setSecurityManager(new NoExitSecurityManager()); @@ -152,7 +171,7 @@ public void testBasicDDLCommands() throws Exception { assertTrue(table1.getOutputFileFormat().equalsIgnoreCase( RCFileOutputFormat.class.getName())); assertTrue(table1.getSerdeLib().equalsIgnoreCase( - ColumnarSerDe.class.getName())); + LazyBinaryColumnarSerDe.class.getName())); assertTrue(table1.getCols().equals(cols)); // Since "ifexists" was not set to true, trying to create the same table // again @@ -171,8 +190,8 @@ public void testBasicDDLCommands() throws Exception { mapKeysTerminatedBy('\004').collectionItemsTerminatedBy('\005').nullDefinedAs('\006').build(); client.createTable(tableDesc2); HCatTable table2 = client.getTable(db, tableTwo); - assertTrue(table2.getInputFileFormat().equalsIgnoreCase( - TextInputFormat.class.getName())); + assertTrue("Expected TextInputFormat, but got: " + table2.getInputFileFormat(), + table2.getInputFileFormat().equalsIgnoreCase(TextInputFormat.class.getName())); assertTrue(table2.getOutputFileFormat().equalsIgnoreCase( HiveIgnoreKeyTextOutputFormat.class.getName())); assertTrue("SerdeParams not found", table2.getSerdeParams() != null); @@ -222,9 +241,10 @@ public void testEmptyTableInstantiation() throws Exception { cols.add(new HCatFieldSchema("id", Type.INT, "id comment")); cols.add(new HCatFieldSchema("value", Type.STRING, "value comment")); + client.dropTable(dbName, tblName, true); // Create a minimalistic table client.createTable(HCatCreateTableDesc - .create(dbName, tblName, cols) + .create(new HCatTable(dbName, tblName).cols(cols), false) .build()); HCatTable tCreated = client.getTable(dbName, tblName); @@ -281,21 +301,26 @@ public void testPartitionsHCatClientImpl() throws Exception { ptnCols.add(new HCatFieldSchema("dt", Type.STRING, "date column")); ptnCols.add(new HCatFieldSchema("country", Type.STRING, "country column")); - HCatCreateTableDesc tableDesc = HCatCreateTableDesc - .create(dbName, tableName, cols).fileFormat("sequencefile") - .partCols(ptnCols).build(); + HCatTable table = new HCatTable(dbName, tableName).cols(cols) + .partCols(ptnCols) + .fileFormat("sequenceFile"); + HCatCreateTableDesc tableDesc = HCatCreateTableDesc.create(table, false).build(); client.createTable(tableDesc); + // Verify that the table is created successfully. + table = client.getTable(dbName, tableName); + Map firstPtn = new HashMap(); firstPtn.put("dt", "04/30/2012"); firstPtn.put("country", "usa"); - HCatAddPartitionDesc addPtn = HCatAddPartitionDesc.create(dbName, - tableName, null, firstPtn).build(); + // Test new HCatAddPartitionsDesc API. + HCatAddPartitionDesc addPtn = HCatAddPartitionDesc.create(new HCatPartition(table, firstPtn, null)).build(); client.addPartition(addPtn); Map secondPtn = new HashMap(); secondPtn.put("dt", "04/12/2012"); secondPtn.put("country", "brazil"); + // Test deprecated HCatAddPartitionsDesc API. HCatAddPartitionDesc addPtn2 = HCatAddPartitionDesc.create(dbName, tableName, null, secondPtn).build(); client.addPartition(addPtn2); @@ -303,6 +328,7 @@ public void testPartitionsHCatClientImpl() throws Exception { Map thirdPtn = new HashMap(); thirdPtn.put("dt", "04/13/2012"); thirdPtn.put("country", "argentina"); + // Test deprecated HCatAddPartitionsDesc API. HCatAddPartitionDesc addPtn3 = HCatAddPartitionDesc.create(dbName, tableName, null, thirdPtn).build(); client.addPartition(addPtn3); @@ -540,9 +566,8 @@ public void testObjectNotFoundException() throws Exception { List columns = Arrays.asList(new HCatFieldSchema("col", Type.STRING, "")); ArrayList partitionColumns = new ArrayList( Arrays.asList(new HCatFieldSchema(partitionColumn, Type.STRING, ""))); - client.createTable(HCatCreateTableDesc.create(dbName, tableName, columns) - .partCols(partitionColumns) - .build()); + HCatTable table = new HCatTable(dbName, tableName).cols(columns).partCols(partitionColumns); + client.createTable(HCatCreateTableDesc.create(table, false).build()); Map partitionSpec = new HashMap(); partitionSpec.put(partitionColumn, "foobar"); @@ -555,7 +580,7 @@ public void testObjectNotFoundException() throws Exception { exception instanceof ObjectNotFoundException); } - client.addPartition(HCatAddPartitionDesc.create(dbName, tableName, "", partitionSpec).build()); + client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table, partitionSpec, "")).build()); // Test that listPartitionsByFilter() returns an empty-set, if the filter selects no partitions. assertEquals("Expected empty set of partitions.", @@ -649,21 +674,26 @@ public void testGetPartitionsWithPartialSpec() throws Exception { List partitionSchema = Arrays.asList(new HCatFieldSchema("dt", Type.STRING, ""), new HCatFieldSchema("grid", Type.STRING, "")); - client.createTable(HCatCreateTableDesc.create(dbName, tableName, columnSchema).partCols(new ArrayList(partitionSchema)).build()); + HCatTable table = new HCatTable(dbName, tableName).cols(columnSchema).partCols(partitionSchema); + client.createTable(HCatCreateTableDesc.create(table, false).build()); + + // Verify that the table was created successfully. + table = client.getTable(dbName, tableName); + assertNotNull("The created just now can't be null.", table); Map partitionSpec = new HashMap(); partitionSpec.put("grid", "AB"); partitionSpec.put("dt", "2011_12_31"); - client.addPartition(HCatAddPartitionDesc.create(dbName, tableName, "", partitionSpec).build()); + client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table, partitionSpec, "")).build()); partitionSpec.put("grid", "AB"); partitionSpec.put("dt", "2012_01_01"); - client.addPartition(HCatAddPartitionDesc.create(dbName, tableName, "", partitionSpec).build()); + client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table, partitionSpec, "")).build()); partitionSpec.put("dt", "2012_01_01"); partitionSpec.put("grid", "OB"); - client.addPartition(HCatAddPartitionDesc.create(dbName, tableName, "", partitionSpec).build()); + client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table, partitionSpec, "")).build()); partitionSpec.put("dt", "2012_01_01"); partitionSpec.put("grid", "XB"); - client.addPartition(HCatAddPartitionDesc.create(dbName, tableName, "", partitionSpec).build()); + client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table, partitionSpec, "")).build()); Map partialPartitionSpec = new HashMap(); partialPartitionSpec.put("dt", "2012_01_01"); @@ -698,21 +728,26 @@ public void testDropPartitionsWithPartialSpec() throws Exception { List partitionSchema = Arrays.asList(new HCatFieldSchema("dt", Type.STRING, ""), new HCatFieldSchema("grid", Type.STRING, "")); - client.createTable(HCatCreateTableDesc.create(dbName, tableName, columnSchema).partCols(new ArrayList(partitionSchema)).build()); + HCatTable table = new HCatTable(dbName, tableName).cols(columnSchema).partCols(partitionSchema); + client.createTable(HCatCreateTableDesc.create(table, false).build()); + + // Verify that the table was created successfully. + table = client.getTable(dbName, tableName); + assertNotNull("Table couldn't be queried for. ", table); Map partitionSpec = new HashMap(); partitionSpec.put("grid", "AB"); partitionSpec.put("dt", "2011_12_31"); - client.addPartition(HCatAddPartitionDesc.create(dbName, tableName, "", partitionSpec).build()); + client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table, partitionSpec, "")).build()); partitionSpec.put("grid", "AB"); partitionSpec.put("dt", "2012_01_01"); - client.addPartition(HCatAddPartitionDesc.create(dbName, tableName, "", partitionSpec).build()); + client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table, partitionSpec, "")).build()); partitionSpec.put("dt", "2012_01_01"); partitionSpec.put("grid", "OB"); - client.addPartition(HCatAddPartitionDesc.create(dbName, tableName, "", partitionSpec).build()); + client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table, partitionSpec, "")).build()); partitionSpec.put("dt", "2012_01_01"); partitionSpec.put("grid", "XB"); - client.addPartition(HCatAddPartitionDesc.create(dbName, tableName, "", partitionSpec).build()); + client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table, partitionSpec, "")).build()); Map partialPartitionSpec = new HashMap(); partialPartitionSpec.put("dt", "2012_01_01"); @@ -731,4 +766,235 @@ public void testDropPartitionsWithPartialSpec() throws Exception { } } + private void startReplicationTargetMetaStoreIfRequired() throws Exception { + if (!isReplicationTargetHCatRunning) { + Thread t = new Thread(new RunMS(replicationTargetHCatPort) + .arg("--hiveconf") + .arg("javax.jdo.option.ConnectionURL") // Reset, to use a different Derby instance. + .arg(hcatConf.get("javax.jdo.option.ConnectionURL") + .replace("metastore", "target_metastore"))); + t.start(); + Thread.sleep(10000); + replicationTargetHCatConf = new HiveConf(hcatConf); + replicationTargetHCatConf.setVar(HiveConf.ConfVars.METASTOREURIS, + "thrift://localhost:" + replicationTargetHCatPort); + isReplicationTargetHCatRunning = true; + } + } + + /** + * Test for detecting schema-changes for an HCatalog table, across 2 different HCat instances. + * A table is created with the same schema on 2 HCat instances. The table-schema is modified on the source HCat + * instance (columns, I/O formats, SerDe definitions, etc.). The table metadata is compared between source + * and target, the changes are detected and propagated to target. + * @throws Exception + */ + @Test + public void testTableSchemaPropagation() throws Exception { + try { + startReplicationTargetMetaStoreIfRequired(); + HCatClient sourceMetaStore = HCatClient.create(new Configuration(hcatConf)); + final String dbName = "myDb"; + final String tableName = "myTable"; + + sourceMetaStore.dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE); + + sourceMetaStore.createDatabase(HCatCreateDBDesc.create(dbName).build()); + List columnSchema = Arrays.asList(new HCatFieldSchema("foo", Type.INT, ""), + new HCatFieldSchema("bar", Type.STRING, "")); + + List partitionSchema = Arrays.asList(new HCatFieldSchema("dt", Type.STRING, ""), + new HCatFieldSchema("grid", Type.STRING, "")); + + HCatTable sourceTable = new HCatTable(dbName, tableName).cols(columnSchema).partCols(partitionSchema); + sourceMetaStore.createTable(HCatCreateTableDesc.create(sourceTable).build()); + + // Verify that the sourceTable was created successfully. + sourceTable = sourceMetaStore.getTable(dbName, tableName); + assertNotNull("Table couldn't be queried for. ", sourceTable); + + // Serialize Table definition. Deserialize using the target HCatClient instance. + String tableStringRep = sourceMetaStore.serializeTable(sourceTable); + HCatClient targetMetaStore = HCatClient.create(new Configuration(replicationTargetHCatConf)); + targetMetaStore.dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE); + targetMetaStore.createDatabase(HCatCreateDBDesc.create(dbName).build()); + + HCatTable targetTable = targetMetaStore.deserializeTable(tableStringRep); + + assertEquals("Table after deserialization should have been identical to sourceTable.", + sourceTable.diff(targetTable), HCatTable.NO_DIFF); + + // Create table on Target. + targetMetaStore.createTable(HCatCreateTableDesc.create(targetTable).build()); + // Verify that the created table is identical to sourceTable. + targetTable = targetMetaStore.getTable(dbName, tableName); + assertEquals("Table after deserialization should have been identical to sourceTable.", + sourceTable.diff(targetTable), HCatTable.NO_DIFF); + + // Modify sourceTable. + List newColumnSchema = new ArrayList(columnSchema); + newColumnSchema.add(new HCatFieldSchema("goo_new", Type.DOUBLE, "")); + Map tableParams = new HashMap(1); + tableParams.put("orc.compress", "ZLIB"); + sourceTable.cols(newColumnSchema) // Add a column. + .fileFormat("orcfile") // Change SerDe, File I/O formats. + .tblProps(tableParams) + .serdeParam(serdeConstants.FIELD_DELIM, Character.toString('\001')); + sourceMetaStore.updateTableSchema(dbName, tableName, sourceTable); + sourceTable = sourceMetaStore.getTable(dbName, tableName); + + // Diff against table on target. + + EnumSet diff = targetTable.diff(sourceTable); + assertTrue("Couldn't find change in column-schema.", + diff.contains(HCatTable.TableAttribute.COLUMNS)); + assertTrue("Couldn't find change in InputFormat.", + diff.contains(HCatTable.TableAttribute.INPUT_FORMAT)); + assertTrue("Couldn't find change in OutputFormat.", + diff.contains(HCatTable.TableAttribute.OUTPUT_FORMAT)); + assertTrue("Couldn't find change in SerDe.", + diff.contains(HCatTable.TableAttribute.SERDE)); + assertTrue("Couldn't find change in SerDe parameters.", + diff.contains(HCatTable.TableAttribute.SERDE_PROPERTIES)); + assertTrue("Couldn't find change in Table parameters.", + diff.contains(HCatTable.TableAttribute.TABLE_PROPERTIES)); + + // Replicate the changes to the replicated-table. + targetMetaStore.updateTableSchema(dbName, tableName, targetTable.resolve(sourceTable, diff)); + targetTable = targetMetaStore.getTable(dbName, tableName); + + assertEquals("After propagating schema changes, source and target tables should have been equivalent.", + targetTable.diff(sourceTable), HCatTable.NO_DIFF); + + } + catch (Exception unexpected) { + LOG.error("Unexpected exception!", unexpected); + assertTrue("Unexpected exception! " + unexpected.getMessage(), false); + } + } + + /** + * Test that partition-definitions can be replicated between HCat-instances, + * independently of table-metadata replication. + * 2 identical tables are created on 2 different HCat instances ("source" and "target"). + * On the source instance, + * 1. One partition is added with the old format ("TEXTFILE"). + * 2. The table is updated with an additional column and the data-format changed to ORC. + * 3. Another partition is added with the new format. + * 4. The partitions' metadata is copied to the target HCat instance, without updating the target table definition. + * 5. The partitions' metadata is tested to be an exact replica of that on the source. + * @throws Exception + */ + @Test + public void testPartitionRegistrationWithCustomSchema() throws Exception { + try { + startReplicationTargetMetaStoreIfRequired(); + + HCatClient sourceMetaStore = HCatClient.create(new Configuration(hcatConf)); + final String dbName = "myDb"; + final String tableName = "myTable"; + + sourceMetaStore.dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE); + + sourceMetaStore.createDatabase(HCatCreateDBDesc.create(dbName).build()); + List columnSchema = new ArrayList( + Arrays.asList(new HCatFieldSchema("foo", Type.INT, ""), + new HCatFieldSchema("bar", Type.STRING, ""))); + + List partitionSchema = Arrays.asList(new HCatFieldSchema("dt", Type.STRING, ""), + new HCatFieldSchema("grid", Type.STRING, "")); + + HCatTable sourceTable = new HCatTable(dbName, tableName).cols(columnSchema) + .partCols(partitionSchema) + .comment("Source table."); + + sourceMetaStore.createTable(HCatCreateTableDesc.create(sourceTable).build()); + + // Verify that the sourceTable was created successfully. + sourceTable = sourceMetaStore.getTable(dbName, tableName); + assertNotNull("Table couldn't be queried for. ", sourceTable); + + // Partitions added now should inherit table-schema, properties, etc. + Map partitionSpec_1 = new HashMap(); + partitionSpec_1.put("grid", "AB"); + partitionSpec_1.put("dt", "2011_12_31"); + HCatPartition sourcePartition_1 = new HCatPartition(sourceTable, partitionSpec_1, ""); + + sourceMetaStore.addPartition(HCatAddPartitionDesc.create(sourcePartition_1).build()); + assertEquals("Unexpected number of partitions. ", + sourceMetaStore.getPartitions(dbName, tableName).size(), 1); + // Verify that partition_1 was added correctly, and properties were inherited from the HCatTable. + HCatPartition addedPartition_1 = sourceMetaStore.getPartition(dbName, tableName, partitionSpec_1); + assertEquals("Column schema doesn't match.", addedPartition_1.getColumns(), sourceTable.getCols()); + assertEquals("InputFormat doesn't match.", addedPartition_1.getInputFormat(), sourceTable.getInputFileFormat()); + assertEquals("OutputFormat doesn't match.", addedPartition_1.getOutputFormat(), sourceTable.getOutputFileFormat()); + assertEquals("SerDe doesn't match.", addedPartition_1.getSerDe(), sourceTable.getSerdeLib()); + assertEquals("SerDe params don't match.", addedPartition_1.getSerdeParams(), sourceTable.getSerdeParams()); + + // Replicate table definition. + + HCatClient targetMetaStore = HCatClient.create(new Configuration(replicationTargetHCatConf)); + targetMetaStore.dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE); + + targetMetaStore.createDatabase(HCatCreateDBDesc.create(dbName).build()); + // Make a copy of the source-table, as would be done across class-loaders. + HCatTable targetTable = targetMetaStore.deserializeTable(sourceMetaStore.serializeTable(sourceTable)); + targetMetaStore.createTable(HCatCreateTableDesc.create(targetTable).build()); + targetTable = targetMetaStore.getTable(dbName, tableName); + + assertEquals("Created table doesn't match the source.", + targetTable.diff(sourceTable), HCatTable.NO_DIFF); + + // Modify Table schema at the source. + List newColumnSchema = new ArrayList(columnSchema); + newColumnSchema.add(new HCatFieldSchema("goo_new", Type.DOUBLE, "")); + Map tableParams = new HashMap(1); + tableParams.put("orc.compress", "ZLIB"); + sourceTable.cols(newColumnSchema) // Add a column. + .fileFormat("orcfile") // Change SerDe, File I/O formats. + .tblProps(tableParams) + .serdeParam(serdeConstants.FIELD_DELIM, Character.toString('\001')); + sourceMetaStore.updateTableSchema(dbName, tableName, sourceTable); + sourceTable = sourceMetaStore.getTable(dbName, tableName); + + // Add another partition to the source. + Map partitionSpec_2 = new HashMap(); + partitionSpec_2.put("grid", "AB"); + partitionSpec_2.put("dt", "2012_01_01"); + HCatPartition sourcePartition_2 = new HCatPartition(sourceTable, partitionSpec_2, ""); + sourceMetaStore.addPartition(HCatAddPartitionDesc.create(sourcePartition_2).build()); + + // The source table now has 2 partitions, one in TEXTFILE, the other in ORC. + // Test that adding these partitions to the target-table *without* replicating the table-change. + + List sourcePartitions = sourceMetaStore.getPartitions(dbName, tableName); + assertEquals("Unexpected number of source partitions.", 2, sourcePartitions.size()); + + List addPartitionDescs = new ArrayList(sourcePartitions.size()); + for (HCatPartition partition : sourcePartitions) { + addPartitionDescs.add(HCatAddPartitionDesc.create(partition).build()); + } + + targetMetaStore.addPartitions(addPartitionDescs); + + List targetPartitions = targetMetaStore.getPartitions(dbName, tableName); + + assertEquals("Expected the same number of partitions. ", targetPartitions.size(), sourcePartitions.size()); + + for (int i=0; i