diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index 33a450585b..67a5ee55e1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -121,6 +121,16 @@ private transient DefaultConstraint dc; private transient CheckConstraint cc; + /** Constraint related flags + * This is to track if constraints are retrieved from metastore or not + */ + private boolean isPKFetched=false; + private boolean isFKFetched=false; + private boolean isUniqueFetched=false; + private boolean isNotNullFetched=false; + private boolean isDefaultFetched=false; + private boolean isCheckFetched=false; + /** * Used only for serialization. */ @@ -147,6 +157,19 @@ public Table(String databaseName, String tableName) { this(getEmptyTable(databaseName, tableName)); } + /** This api is used by getMetaData which require deep copy of metastore.api.table + * and constraints copy + */ + public Table makeCopy() { + + // make deep copy of metastore.api.table + Table newTab = new Table(this.getTTable().deepCopy()); + + // copy constraints + newTab.copyConstraints(this); + return newTab; + } + public boolean isDummyTable() { return tTable.getTableName().equals(SemanticAnalyzer.DUMMY_TABLE); } @@ -171,7 +194,7 @@ public void setTTable(org.apache.hadoop.hive.metastore.api.Table tTable) { * Initialize an empty table. */ public static org.apache.hadoop.hive.metastore.api.Table - getEmptyTable(String databaseName, String tableName) { + getEmptyTable(String databaseName, String tableName) { StorageDescriptor sd = new StorageDescriptor(); { sd.setSerdeInfo(new SerDeInfo()); @@ -316,8 +339,8 @@ public HiveStorageHandler getStorageHandler() { try { storageHandler = HiveUtils.getStorageHandler( SessionState.getSessionConf(), - getProperty( - org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE)); + getProperty( + org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE)); } catch (Exception e) { throw new RuntimeException(e); } @@ -343,7 +366,7 @@ public void setStorageHandlerInfo(StorageHandlerInfo storageHandlerInfo) { inputFormatClass = getStorageHandler().getInputFormatClass(); } else { inputFormatClass = (Class) - Class.forName(className, true, Utilities.getSessionSpecifiedClassLoader()); + Class.forName(className, true, Utilities.getSessionSpecifiedClassLoader()); } } catch (ClassNotFoundException e) { throw new RuntimeException(e); @@ -449,12 +472,12 @@ public boolean isImmutable(){ } public void setTableType(TableType tableType) { - tTable.setTableType(tableType.toString()); - } + tTable.setTableType(tableType.toString()); + } public TableType getTableType() { - return Enum.valueOf(TableType.class, tTable.getTableType()); - } + return Enum.valueOf(TableType.class, tTable.getTableType()); + } public ArrayList getFields() { @@ -491,46 +514,46 @@ public int getBucketingVersion() { getProperty(hive_metastoreConstants.TABLE_BUCKETING_VERSION)); } - @Override + @Override public String toString() { return tTable.getTableName(); } - /* (non-Javadoc) - * @see java.lang.Object#hashCode() - */ - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((tTable == null) ? 0 : tTable.hashCode()); - return result; - } - - /* (non-Javadoc) - * @see java.lang.Object#equals(java.lang.Object) - */ - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - Table other = (Table) obj; - if (tTable == null) { - if (other.tTable != null) { - return false; - } - } else if (!tTable.equals(other.tTable)) { - return false; - } - return true; - } + /* (non-Javadoc) + * @see java.lang.Object#hashCode() + */ + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((tTable == null) ? 0 : tTable.hashCode()); + return result; + } + + /* (non-Javadoc) + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Table other = (Table) obj; + if (tTable == null) { + if (other.tTable != null) { + return false; + } + } else if (!tTable.equals(other.tTable)) { + return false; + } + return true; + } public List getPartCols() { @@ -596,7 +619,7 @@ public void setBucketCols(List bucketCols) throws HiveException { for (String col : bucketCols) { if (!isField(col)) { throw new HiveException("Bucket columns " + col - + " is not part of the table columns (" + getCols() ); + + " is not part of the table columns (" + getCols()); } } tTable.getSd().setBucketCols(bucketCols); @@ -619,7 +642,7 @@ public void setSkewedValueLocationMap(List valList, String dirName) mappings.put(valList, dirName); } - public Map,String> getSkewedColValueLocationMaps() { + public Map, String> getSkewedColValueLocationMaps() { return (tTable.getSd().getSkewedInfo() != null) ? tTable.getSd().getSkewedInfo() .getSkewedColValueLocationMaps() : new HashMap, String>(); } @@ -977,8 +1000,8 @@ public void setLastAccessTime(int lastAccessTime) { public boolean isNonNative() { return getProperty( - org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE) - != null; + org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE) + != null; } public String getFullyQualifiedName() { @@ -1057,7 +1080,7 @@ public static boolean shouldStoreFieldsInMetastore( try { Class clazz = conf.getClassByName(serdeLib); if (!AbstractSerDe.class.isAssignableFrom(clazz)) - { + { return true; // The default. } deserializer = ReflectionUtil.newInstance( @@ -1126,71 +1149,167 @@ public Boolean isOutdatedForRewriting() { return outdatedForRewritingMaterializedView; } - /* These are only populated during optimization and describing */ + public ColumnStatistics getColStats() { + return tTable.isSetColStats() ? tTable.getColStats() : null; + } + + /** + * Setup the table level stats as if the table is new. Used when setting up Table for a new + * table or during replication. + */ + public void setStatsStateLikeNewTable() { + if (isPartitioned()) { + StatsSetupConst.setStatsStateForCreateTable(getParameters(), null, + StatsSetupConst.FALSE); + } else { + StatsSetupConst.setStatsStateForCreateTable(getParameters(), + MetaStoreUtils.getColumnNames(getCols()), StatsSetupConst.TRUE); + } + } + + /** Constraints related methods + * Note that set apis are used by DESCRIBE only, although get apis return RELY or ENABLE + * constraints DESCRIBE could set all type of constraints + * */ + + /* This only return PK with RELY */ public PrimaryKeyInfo getPrimaryKeyInfo() { + if(!this.isPKFetched) { + try { + pki = Hive.get().getReliablePrimaryKeys(this.getDbName(), this.getTableName()); + this.isPKFetched = true; + } catch (HiveException e) { + LOG.warn("Cannot retrieve PK info for table : " + this.getTableName() + + " ignoring exception: " + e); + } + } return pki; } public void setPrimaryKeyInfo(PrimaryKeyInfo pki) { this.pki = pki; + this.isPKFetched = true; } + /* This only return FK with RELY */ public ForeignKeyInfo getForeignKeyInfo() { + if(!isFKFetched) { + try { + fki = Hive.get().getReliableForeignKeys(this.getDbName(), this.getTableName()); + this.isFKFetched = true; + } catch (HiveException e) { + LOG.warn( + "Cannot retrieve FK info for table : " + this.getTableName() + + " ignoring exception: " + e); + } + } return fki; } public void setForeignKeyInfo(ForeignKeyInfo fki) { this.fki = fki; + this.isFKFetched = true; } + /* This only return UNIQUE constraint defined with RELY */ public UniqueConstraint getUniqueKeyInfo() { + if(!isUniqueFetched) { + try { + uki = Hive.get().getReliableUniqueConstraints(this.getDbName(), this.getTableName()); + this.isUniqueFetched = true; + } catch (HiveException e) { + LOG.warn( + "Cannot retrieve Unique Key info for table : " + this.getTableName() + + " ignoring exception: " + e); + } + } return uki; } public void setUniqueKeyInfo(UniqueConstraint uki) { this.uki = uki; + this.isUniqueFetched = true; } + /* This only return NOT NULL constraint defined with RELY */ public NotNullConstraint getNotNullConstraint() { + if(!isNotNullFetched) { + try { + nnc = Hive.get().getReliableNotNullConstraints(this.getDbName(), this.getTableName()); + this.isNotNullFetched = true; + } catch (HiveException e) { + LOG.warn("Cannot retrieve Not Null constraint info for table : " + + this.getTableName() + " ignoring exception: " + e); + } + } return nnc; } public void setNotNullConstraint(NotNullConstraint nnc) { this.nnc = nnc; + this.isNotNullFetched = true; } + /* This only return DEFAULT constraint defined with ENABLE */ public DefaultConstraint getDefaultConstraint() { + if(!isDefaultFetched) { + try { + dc = Hive.get().getEnabledDefaultConstraints(this.getDbName(), this.getTableName()); + this.isDefaultFetched = true; + } catch (HiveException e) { + LOG.warn("Cannot retrieve Default constraint info for table : " + + this.getTableName() + " ignoring exception: " + e); + } + } return dc; } public void setDefaultConstraint(DefaultConstraint dc) { this.dc = dc; + this.isDefaultFetched = true; } + /* This only return CHECK constraint defined with ENABLE */ public CheckConstraint getCheckConstraint() { + if(!isCheckFetched) { + try{ + cc = Hive.get().getEnabledCheckConstraints(this.getDbName(), this.getTableName()); + this.isCheckFetched = true; + } catch (HiveException e) { + LOG.warn("Cannot retrieve Check constraint info for table : " + + this.getTableName() + " ignoring exception: " + e); + } + } return cc; } public void setCheckConstraint(CheckConstraint cc) { this.cc = cc; + this.isCheckFetched = true; } + /** This shouldn't use get apis because those api call metastore + * to fetch constraints. + * getMetaData only need to make a copy of existing constraints, even if those are not fetched + */ + public void copyConstraints(final Table tbl) { + this.pki = tbl.pki; + this.isPKFetched = tbl.isPKFetched; - public ColumnStatistics getColStats() { - return tTable.isSetColStats() ? tTable.getColStats() : null; - } + this.fki = tbl.fki; + this.isFKFetched = tbl.isFKFetched; - /** - * Setup the table level stats as if the table is new. Used when setting up Table for a new - * table or during replication. - */ - public void setStatsStateLikeNewTable() { - if (isPartitioned()) { - StatsSetupConst.setStatsStateForCreateTable(getParameters(), null, - StatsSetupConst.FALSE); - } else { - StatsSetupConst.setStatsStateForCreateTable(getParameters(), - MetaStoreUtils.getColumnNames(getCols()), StatsSetupConst.TRUE); - } + this.uki = tbl.uki; + this.isUniqueFetched = tbl.isUniqueFetched; + + this.nnc = tbl.nnc; + this.isNotNullFetched = tbl.isNotNullFetched; + + this.dc = tbl.dc; + this.isDefaultFetched = tbl.isDefaultFetched; + + this.cc = tbl.cc; + this.isCheckFetched = tbl.isCheckFetched; } + }; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index cde8eada64..42d243f670 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -5222,24 +5222,19 @@ private QBParseInfo getQBParseInfo(QB qb) throws CalciteSemanticException { } @Override - protected Table getTableObjectByName(String tableName, boolean throwException) throws HiveException { - if (!tabNameToTabObject.containsKey(tableName)) { - // TODO: The code below should be a single HMS call and possibly unified with method in SemanticAnalyzer - Table table = db.getTable(tableName, throwException); + protected Table getTableObjectByName(String tabName, boolean throwException) throws HiveException { + String[] names = Utilities.getDbTableName(tabName); + final String tableName = names[1]; + final String dbName = names[0]; + final String fullyQualName = dbName + "." + tableName; + if (!tabNameToTabObject.containsKey(fullyQualName)) { + Table table = db.getTable(dbName, tableName, throwException); if (table != null) { - table.setPrimaryKeyInfo(db.getReliablePrimaryKeys( - table.getDbName(), table.getTableName())); - table.setForeignKeyInfo(db.getReliableForeignKeys( - table.getDbName(), table.getTableName())); - table.setUniqueKeyInfo(db.getReliableUniqueConstraints( - table.getDbName(), table.getTableName())); - table.setNotNullConstraint(db.getReliableNotNullConstraints( - table.getDbName(), table.getTableName())); - tabNameToTabObject.put(tableName, table); + tabNameToTabObject.put(fullyQualName, table); } return table; } - return tabNameToTabObject.get(tableName); + return tabNameToTabObject.get(fullyQualName); } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index f5df9cd302..ad24ac53f0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -487,7 +487,6 @@ protected void reset(boolean clearCache) { } else { mergeIsDirect = false; } - tabNameToTabObject.clear(); loadTableWork.clear(); loadFileWork.clear(); columnStatsAutoGatherContexts.clear(); @@ -2139,14 +2138,7 @@ private void getMetaData(QB qb, ReadEntity parentInput) // Get table details from tabNameToTabObject cache Table tab = getTableObjectByName(tabName, false); if (tab != null) { - // copy table object in case downstream changes it - Table newTab = new Table(tab.getTTable().deepCopy()); - // copy constraints, we do not need deep copy as - // they should not be changed - newTab.setPrimaryKeyInfo(tab.getPrimaryKeyInfo()); - newTab.setForeignKeyInfo(tab.getForeignKeyInfo()); - newTab.setUniqueKeyInfo(tab.getUniqueKeyInfo()); - newTab.setNotNullConstraint(tab.getNotNullConstraint()); + Table newTab = tab.makeCopy(); tab = newTab; } if (tab == null || diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java index 8084dcd62a..cb2d0a7a7e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java @@ -1024,7 +1024,7 @@ else if(colTypeLowerCase.equals(serdeConstants.SMALLINT_TYPE_NAME)){ // This table is dummy and has no stats return stats; } - if (fetchColStats) { + if (fetchColStats && !colStatsToRetrieve.isEmpty()) { try { List colStat = Hive.get().getTableColumnStatistics( dbName, tabName, colStatsToRetrieve, false); diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index cc97418d16..9c95223afc 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -2565,6 +2565,9 @@ public void flushCache() { @Override public List getTableColumnStatistics(String catName, String dbName, String tableName, List colNames, String engine) throws TException { + if(colNames.isEmpty()) { + return Collections.emptyList(); + } TableStatsRequest rqst = new TableStatsRequest(dbName, tableName, colNames, engine); rqst.setCatName(catName); rqst.setEngine(engine); @@ -2581,6 +2584,9 @@ public void flushCache() { @Override public List getTableColumnStatistics(String catName, String dbName, String tableName, List colNames, String engine, String validWriteIdList) throws TException { + if(colNames.isEmpty()) { + return Collections.emptyList(); + } TableStatsRequest rqst = new TableStatsRequest(dbName, tableName, colNames, engine); rqst.setCatName(catName); rqst.setEngine(engine);