diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index ae030d99e7..aba30359ba 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -112,6 +112,12 @@ * a materialization is up-to-date or not. */ private transient Boolean outdatedForRewritingMaterializedView; + /** Constraint related objects */ + private transient PrimaryKeyInfo pki; + private transient ForeignKeyInfo fki; + private transient UniqueConstraint uki; + private transient NotNullConstraint nnc; + /** * Used only for serialization. */ @@ -1109,6 +1115,40 @@ public Boolean isOutdatedForRewriting() { return outdatedForRewritingMaterializedView; } + /* These are only populated during optimization */ + public PrimaryKeyInfo getPrimaryKeyInfo() { + return pki; + } + + public void setPrimaryKeyInfo(PrimaryKeyInfo pki) { + this.pki = pki; + } + + public ForeignKeyInfo getForeignKeyInfo() { + return fki; + } + + public void setForeignKeyInfo(ForeignKeyInfo fki) { + this.fki = fki; + } + + public UniqueConstraint getUniqueKeyInfo() { + return uki; + } + + public void setUniqueKeyInfo(UniqueConstraint uki) { + this.uki = uki; + } + + public NotNullConstraint getNotNullConstraint() { + return nnc; + } + + public void setNotNullConstraint(NotNullConstraint nnc) { + this.nnc = nnc; + } + + public ColumnStatistics getColStats() { return tTable.isSetColStats() ? tTable.getColStats() : null; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java index 995ff9b68c..58e277133c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java @@ -101,7 +101,7 @@ private final List keys; private final List nonNullablekeys; private final List referentialConstraints; - final HiveConf hiveConf; + private final HiveConf hiveConf; private double rowCount = -1; Map hiveColStatsMap = new HashMap<>(); @@ -112,11 +112,11 @@ protected static final Logger LOG = LoggerFactory.getLogger(RelOptHiveTable.class.getName()); + public RelOptHiveTable(RelOptSchema calciteSchema, RelDataTypeFactory typeFactory, List qualifiedTblName, - RelDataType rowType, Table hiveTblMetadata, List hiveNonPartitionCols, - List hivePartitionCols, List hiveVirtualCols, HiveConf hconf, - Map partitionCache, Map colStatsCache, - AtomicInteger noColsMissingStats) { + RelDataType rowType, Table hiveTblMetadata, List hiveNonPartitionCols, List hivePartitionCols, + List hiveVirtualCols, HiveConf hconf, Map partitionCache, + Map colStatsCache, AtomicInteger noColsMissingStats) { this.schema = calciteSchema; this.typeFactory = typeFactory; this.qualifiedTblName = ImmutableList.copyOf(qualifiedTblName); @@ -245,19 +245,14 @@ public boolean isKey(ImmutableBitSet columns) { } private Pair, List> generateKeys() { + final PrimaryKeyInfo primaryKeyInfo = hiveTblMetadata.getPrimaryKeyInfo(); + final UniqueConstraint uniqueKeyInfo = hiveTblMetadata.getUniqueKeyInfo(); ImmutableList.Builder builder = ImmutableList.builder(); ImmutableList.Builder nonNullbuilder = ImmutableList.builder(); // First PK - final PrimaryKeyInfo pki; - try { - pki = Hive.get().getReliablePrimaryKeys( - hiveTblMetadata.getDbName(), hiveTblMetadata.getTableName()); - } catch (HiveException e) { - throw new RuntimeException(e); - } - if (!pki.getColNames().isEmpty()) { + if (primaryKeyInfo != null && !primaryKeyInfo.getColNames().isEmpty()) { ImmutableBitSet.Builder keys = ImmutableBitSet.builder(); - for (String pkColName : pki.getColNames().values()) { + for (String pkColName : primaryKeyInfo.getColNames().values()) { int pkPos; for (pkPos = 0; pkPos < rowType.getFieldNames().size(); pkPos++) { String colName = rowType.getFieldNames().get(pkPos); @@ -275,99 +270,90 @@ public boolean isKey(ImmutableBitSet columns) { nonNullbuilder.add(key); } // Then UKs - final UniqueConstraint uki; - try { - uki = Hive.get().getReliableUniqueConstraints( - hiveTblMetadata.getDbName(), hiveTblMetadata.getTableName()); - } catch (HiveException e) { - throw new RuntimeException(e); - } - for (List ukCols : uki.getUniqueConstraints().values()) { - ImmutableBitSet.Builder keys = ImmutableBitSet.builder(); - boolean isNonNullable = true; - for (UniqueConstraintCol ukCol : ukCols) { - int ukPos; - for (ukPos = 0; ukPos < rowType.getFieldNames().size(); ukPos++) { - String colName = rowType.getFieldNames().get(ukPos); - if (ukCol.colName.equals(colName)) { - if(rowType.getFieldList().get(ukPos).getType().isNullable()) { - // they should all be nullable - isNonNullable = false; + if (uniqueKeyInfo != null && !uniqueKeyInfo.getUniqueConstraints().isEmpty()) { + for (List ukCols : uniqueKeyInfo.getUniqueConstraints().values()) { + ImmutableBitSet.Builder keys = ImmutableBitSet.builder(); + boolean isNonNullable = true; + for (UniqueConstraintCol ukCol : ukCols) { + int ukPos; + for (ukPos = 0; ukPos < rowType.getFieldNames().size(); ukPos++) { + String colName = rowType.getFieldNames().get(ukPos); + if (ukCol.colName.equals(colName)) { + if (rowType.getFieldList().get(ukPos).getType().isNullable()) { + // they should all be nullable + isNonNullable = false; + } + break; } - break; } + if (ukPos == rowType.getFieldNames().size()) { + LOG.error("Column for unique constraint definition " + ukCol.colName + " not found"); + } + keys.set(ukPos); } - if (ukPos == rowType.getFieldNames().size()) { - LOG.error("Column for unique constraint definition " + ukCol.colName + " not found"); + ImmutableBitSet key = keys.build(); + builder.add(key); + if (isNonNullable) { + nonNullbuilder.add(key); } - keys.set(ukPos); - } - ImmutableBitSet key = keys.build(); - builder.add(key); - if(isNonNullable) { - nonNullbuilder.add(key); } } return new Pair<>(builder.build(), nonNullbuilder.build()); } private List generateReferentialConstraints() { - final ForeignKeyInfo fki; - try { - fki = Hive.get().getReliableForeignKeys( - hiveTblMetadata.getDbName(), hiveTblMetadata.getTableName()); - } catch (HiveException e) { - throw new RuntimeException(e); - } + final ForeignKeyInfo foreignKeyInfo = hiveTblMetadata.getForeignKeyInfo(); ImmutableList.Builder builder = ImmutableList.builder(); - for (List fkCols : fki.getForeignKeys().values()) { - List foreignKeyTableQualifiedName = qualifiedTblName; - String parentDatabaseName = fkCols.get(0).parentDatabaseName; - String parentTableName = fkCols.get(0).parentTableName; - List parentTableQualifiedName = new ArrayList<>(); - if (parentDatabaseName != null && !parentDatabaseName.isEmpty()) { - parentTableQualifiedName.add(parentDatabaseName); - } - parentTableQualifiedName.add(parentTableName); - Table parentTab = null; - try { - // TODO: We have a cache for Table objects in SemanticAnalyzer::getTableObjectByName() - // We need to move that cache elsewhere and use it from places like this. - parentTab = Hive.get().getTable(parentDatabaseName, parentTableName); - } catch (HiveException e) { - throw new RuntimeException(e); - } - if (parentTab == null) { - LOG.error("Table for primary key not found: " - + "databaseName: " + parentDatabaseName+ ", " - + "tableName: " + parentTableName); - return ImmutableList.of(); - } - ImmutableList.Builder keys = ImmutableList.builder(); - for (ForeignKeyCol fkCol : fkCols) { - int fkPos; - for (fkPos = 0; fkPos < rowType.getFieldNames().size(); fkPos++) { - String fkColName = rowType.getFieldNames().get(fkPos); - if (fkColName.equals(fkCol.childColName)) { - break; - } + if (foreignKeyInfo != null && !foreignKeyInfo.getForeignKeys().isEmpty()) { + for (List fkCols : foreignKeyInfo.getForeignKeys().values()) { + List foreignKeyTableQualifiedName = qualifiedTblName; + String parentDatabaseName = fkCols.get(0).parentDatabaseName; + String parentTableName = fkCols.get(0).parentTableName; + List parentTableQualifiedName = new ArrayList<>(); + if (parentDatabaseName != null && !parentDatabaseName.isEmpty()) { + parentTableQualifiedName.add(parentDatabaseName); } - int pkPos; - for (pkPos = 0; pkPos < parentTab.getAllCols().size(); pkPos++) { - String pkColName = parentTab.getAllCols().get(pkPos).getName(); - if (pkColName.equals(fkCol.parentColName)) { - break; - } + parentTableQualifiedName.add(parentTableName); + Table parentTab = null; + try { + // TODO: We have a cache for Table objects in SemanticAnalyzer::getTableObjectByName() + // We need to move that cache elsewhere and use it from places like this. + parentTab = Hive.get().getTable(parentDatabaseName, parentTableName); + } catch (HiveException e) { + throw new RuntimeException(e); } - if (fkPos == rowType.getFieldNames().size() - || pkPos == parentTab.getAllCols().size()) { - LOG.error("Column for foreign key definition " + fkCol + " not found"); + if (parentTab == null) { + LOG.error("Table for primary key not found: " + + "databaseName: " + parentDatabaseName + ", " + + "tableName: " + parentTableName); return ImmutableList.of(); } - keys.add(IntPair.of(fkPos, pkPos)); + ImmutableList.Builder keys = ImmutableList.builder(); + for (ForeignKeyCol fkCol : fkCols) { + int fkPos; + for (fkPos = 0; fkPos < rowType.getFieldNames().size(); fkPos++) { + String fkColName = rowType.getFieldNames().get(fkPos); + if (fkColName.equals(fkCol.childColName)) { + break; + } + } + int pkPos; + for (pkPos = 0; pkPos < parentTab.getAllCols().size(); pkPos++) { + String pkColName = parentTab.getAllCols().get(pkPos).getName(); + if (pkColName.equals(fkCol.parentColName)) { + break; + } + } + if (fkPos == rowType.getFieldNames().size() + || pkPos == parentTab.getAllCols().size()) { + LOG.error("Column for foreign key definition " + fkCol + " not found"); + return ImmutableList.of(); + } + keys.add(IntPair.of(fkPos, pkPos)); + } + builder.add(RelReferentialConstraintImpl.of(foreignKeyTableQualifiedName, + parentTableQualifiedName, keys.build())); } - builder.add(RelReferentialConstraintImpl.of(foreignKeyTableQualifiedName, - parentTableQualifiedName, keys.build())); } return builder.build(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 4011d99c92..f4f5e8d788 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -135,7 +135,6 @@ import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.log.PerfLogger; -import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.NotNullConstraint; import org.apache.hadoop.hive.ql.metadata.PrimaryKeyInfo; @@ -3083,45 +3082,46 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc private RelDataType inferNotNullableColumns(Table tabMetaData, RelDataType rowType) throws HiveException { - // Retrieve not null constraints - final NotNullConstraint nnc = Hive.get().getReliableNotNullConstraints( - tabMetaData.getDbName(), tabMetaData.getTableName()); - // Retrieve primary key constraints (cannot be null) - final PrimaryKeyInfo pkc = Hive.get().getReliablePrimaryKeys( - tabMetaData.getDbName(), tabMetaData.getTableName()); - if (nnc.getNotNullConstraints().isEmpty() && pkc.getColNames().isEmpty()) { + final NotNullConstraint nnc = tabMetaData.getNotNullConstraint(); + final PrimaryKeyInfo pkc = tabMetaData.getPrimaryKeyInfo(); + if ((nnc == null || nnc.getNotNullConstraints().isEmpty()) && + (pkc == null || pkc.getColNames().isEmpty())) { return rowType; } // Build the bitset with not null columns ImmutableBitSet.Builder builder = ImmutableBitSet.builder(); - for (String nnCol : nnc.getNotNullConstraints().values()) { - int nnPos = -1; - for (int i = 0; i < rowType.getFieldNames().size(); i++) { - if (rowType.getFieldNames().get(i).equals(nnCol)) { - nnPos = i; - break; + if (nnc != null) { + for (String nnCol : nnc.getNotNullConstraints().values()) { + int nnPos = -1; + for (int i = 0; i < rowType.getFieldNames().size(); i++) { + if (rowType.getFieldNames().get(i).equals(nnCol)) { + nnPos = i; + break; + } } + if (nnPos == -1) { + LOG.error("Column for not null constraint definition " + nnCol + " not found"); + return rowType; + } + builder.set(nnPos); } - if (nnPos == -1) { - LOG.error("Column for not null constraint definition " + nnCol + " not found"); - return rowType; - } - builder.set(nnPos); } - for (String pkCol : pkc.getColNames().values()) { - int pkPos = -1; - for (int i = 0; i < rowType.getFieldNames().size(); i++) { - if (rowType.getFieldNames().get(i).equals(pkCol)) { - pkPos = i; - break; + if (pkc != null) { + for (String pkCol : pkc.getColNames().values()) { + int pkPos = -1; + for (int i = 0; i < rowType.getFieldNames().size(); i++) { + if (rowType.getFieldNames().get(i).equals(pkCol)) { + pkPos = i; + break; + } } + if (pkPos == -1) { + LOG.error("Column for not null constraint definition " + pkCol + " not found"); + return rowType; + } + builder.set(pkPos); } - if (pkPos == -1) { - LOG.error("Column for not null constraint definition " + pkCol + " not found"); - return rowType; - } - builder.set(pkPos); } ImmutableBitSet bitSet = builder.build(); @@ -5172,6 +5172,27 @@ private QBParseInfo getQBParseInfo(QB qb) throws CalciteSemanticException { } } + @Override + protected Table getTableObjectByName(String tableName, boolean throwException) throws HiveException { + if (!tabNameToTabObject.containsKey(tableName)) { + // TODO: The code below should be a single HMS call and possibly unified with method in SemanticAnalyzer + Table table = db.getTable(tableName, throwException); + if (table != null) { + table.setPrimaryKeyInfo(db.getReliablePrimaryKeys( + table.getDbName(), table.getTableName())); + table.setForeignKeyInfo(db.getReliableForeignKeys( + table.getDbName(), table.getTableName())); + table.setUniqueKeyInfo(db.getReliableUniqueConstraints( + table.getDbName(), table.getTableName())); + table.setNotNullConstraint(db.getReliableNotNullConstraints( + table.getDbName(), table.getTableName())); + tabNameToTabObject.put(tableName, table); + } + return table; + } + return tabNameToTabObject.get(tableName); + } + /** * This method can be called at startup time to pre-register all the * additional Hive classes (compared to Calcite core classes) that may diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index edb20ae7e6..655111ed8b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -1824,7 +1824,7 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PlannerContext plan } Table table = null; try { - table = this.getTableObjectByName(tableName); + table = getTableObjectByName(tableName); } catch (HiveException ex) { throw new SemanticException(ex); } @@ -11996,8 +11996,8 @@ protected String rewriteQueryWithQualifiedNames(ASTNode ast, TokenRewriteStream return rewrittenQuery; } - private static void walkASTMarkTABREF(TableMask tableMask, ASTNode ast, Set cteAlias, - Context ctx, Hive db, Map tabNameToTabObject, Set ignoredTokens) + private void walkASTMarkTABREF(TableMask tableMask, ASTNode ast, Set cteAlias, + Context ctx, Hive db, Map tabNameToTabObject, Set ignoredTokens) throws SemanticException { Queue queue = new LinkedList<>(); queue.add(ast); @@ -12039,9 +12039,16 @@ private static void walkASTMarkTABREF(TableMask tableMask, ASTNode ast, Set tabNameToTabObject) { - Table table = null; - try { - if (!tabNameToTabObject.containsKey(tabIdName)) { - table = db.getTable(tabIdName, true); - tabNameToTabObject.put(tabIdName, table); - } else { - table = tabNameToTabObject.get(tabIdName); - } - } catch (HiveException e) { - // Table may not be found when materialization of CTE is on. - STATIC_LOG.debug("Table " + tabIdName + " is not found in walkASTMarkTABREF."); - } - return table; - } - private static void extractColumnInfos(Table table, List colNames, List colTypes) { for (FieldSchema col : table.getAllCols()) { colNames.add(col.getName()); @@ -12134,8 +12126,8 @@ private static void extractColumnInfos(Table table, List colNames, List< // the table needs to be masked or filtered. // For the replacement, we leverage the methods that are used for // unparseTranslator. - protected static ASTNode rewriteASTWithMaskAndFilter(TableMask tableMask, ASTNode ast, TokenRewriteStream tokenRewriteStream, - Context ctx, Hive db, Map tabNameToTabObject, Set ignoredTokens) + protected ASTNode rewriteASTWithMaskAndFilter(TableMask tableMask, ASTNode ast, TokenRewriteStream tokenRewriteStream, + Context ctx, Hive db, Map tabNameToTabObject, Set ignoredTokens) throws SemanticException { // 1. collect information about CTE if there is any. // The base table of CTE should be masked. @@ -13809,7 +13801,7 @@ private void validateCreateView() if (DUMMY_TABLE.equals(alias)) { continue; } - Table table = this.getTableObjectByName(qb.getTabNameForAlias(alias)); + Table table = getTableObjectByName(qb.getTabNameForAlias(alias)); if (table.isTemporary()) { throw new SemanticException("View definition references temporary table " + alias); } @@ -14039,7 +14031,7 @@ private void validateAnalyzeNoscan(ASTNode tree) throws SemanticException { String tableName = getUnescapedName((ASTNode) tree.getChild(0).getChild(0)); Table tbl; try { - tbl = this.getTableObjectByName(tableName); + tbl = getTableObjectByName(tableName); } catch (InvalidTableException e) { throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(tableName), e); }