diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java index 56047cfa0b..d90c9d0763 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java @@ -379,7 +379,8 @@ private static RelNode createMaterializedViewScan(HiveConf conf, Table viewTable List intervals = Arrays.asList(DruidTable.DEFAULT_INTERVAL); rowType = dtFactory.createStructType(druidColTypes, druidColNames); RelOptHiveTable optTable = new RelOptHiveTable(null, cluster.getTypeFactory(), fullyQualifiedTabName, - rowType, viewTable, nonPartitionColumns, partitionColumns, new ArrayList<>(), + rowType, viewTable, null, null, null, + nonPartitionColumns, partitionColumns, new ArrayList<>(), conf, new HashMap<>(), new HashMap<>(), new AtomicInteger()); DruidTable druidTable = new DruidTable(new DruidSchema(address, address, false), dataSource, RelDataTypeImpl.proto(rowType), metrics, DruidTable.DEFAULT_TIMESTAMP_COLUMN, @@ -391,7 +392,8 @@ private static RelNode createMaterializedViewScan(HiveConf conf, Table viewTable } else { // Build Hive Table Scan Rel RelOptHiveTable optTable = new RelOptHiveTable(null, cluster.getTypeFactory(), fullyQualifiedTabName, - rowType, viewTable, nonPartitionColumns, partitionColumns, new ArrayList<>(), + rowType, viewTable, null, null, null, + nonPartitionColumns, partitionColumns, new ArrayList<>(), conf, new HashMap<>(), new HashMap<>(), new AtomicInteger()); tableRel = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), optTable, viewTable.getTableName(), null, false, false); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java index 995ff9b68c..d9dd02f283 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java @@ -92,6 +92,9 @@ private final List qualifiedTblName; private final String name; private final Table hiveTblMetadata; + private final PrimaryKeyInfo primaryKeyInfo; + private final ForeignKeyInfo foreignKeyInfo; + private final UniqueConstraint uniqueKeyInfo; private final ImmutableList hiveNonPartitionCols; private final ImmutableList hivePartitionCols; private final ImmutableMap hiveNonPartitionColsMap; @@ -101,7 +104,7 @@ private final List keys; private final List nonNullablekeys; private final List referentialConstraints; - final HiveConf hiveConf; + private final HiveConf hiveConf; private double rowCount = -1; Map hiveColStatsMap = new HashMap<>(); @@ -112,17 +115,21 @@ protected static final Logger LOG = LoggerFactory.getLogger(RelOptHiveTable.class.getName()); + public RelOptHiveTable(RelOptSchema calciteSchema, RelDataTypeFactory typeFactory, List qualifiedTblName, - RelDataType rowType, Table hiveTblMetadata, List hiveNonPartitionCols, - List hivePartitionCols, List hiveVirtualCols, HiveConf hconf, - Map partitionCache, Map colStatsCache, - AtomicInteger noColsMissingStats) { + RelDataType rowType, Table hiveTblMetadata, PrimaryKeyInfo primaryKeyInfo, ForeignKeyInfo foreignKeyInfo, + UniqueConstraint uniqueKeyInfo, List hiveNonPartitionCols, List hivePartitionCols, + List hiveVirtualCols, HiveConf hconf, Map partitionCache, + Map colStatsCache, AtomicInteger noColsMissingStats) { this.schema = calciteSchema; this.typeFactory = typeFactory; this.qualifiedTblName = ImmutableList.copyOf(qualifiedTblName); this.name = this.qualifiedTblName.stream().collect(Collectors.joining(".")); this.rowType = rowType; this.hiveTblMetadata = hiveTblMetadata; + this.primaryKeyInfo = primaryKeyInfo; + this.foreignKeyInfo = foreignKeyInfo; + this.uniqueKeyInfo = uniqueKeyInfo; this.hiveNonPartitionCols = ImmutableList.copyOf(hiveNonPartitionCols); this.hiveNonPartitionColsMap = HiveCalciteUtil.getColInfoMap(hiveNonPartitionCols, 0); this.hivePartitionCols = ImmutableList.copyOf(hivePartitionCols); @@ -215,7 +222,8 @@ public RelOptHiveTable copy(RelDataType newRowType) { // 3. Build new Table return new RelOptHiveTable(this.schema, this.typeFactory, this.qualifiedTblName, newRowType, - this.hiveTblMetadata, newHiveNonPartitionCols, newHivePartitionCols, newHiveVirtualCols, + this.hiveTblMetadata, this.primaryKeyInfo, this.foreignKeyInfo, this.uniqueKeyInfo, + newHiveNonPartitionCols, newHivePartitionCols, newHiveVirtualCols, this.hiveConf, this.partitionCache, this.colStatsCache, this.noColsMissingStats); } @@ -248,16 +256,9 @@ public boolean isKey(ImmutableBitSet columns) { ImmutableList.Builder builder = ImmutableList.builder(); ImmutableList.Builder nonNullbuilder = ImmutableList.builder(); // First PK - final PrimaryKeyInfo pki; - try { - pki = Hive.get().getReliablePrimaryKeys( - hiveTblMetadata.getDbName(), hiveTblMetadata.getTableName()); - } catch (HiveException e) { - throw new RuntimeException(e); - } - if (!pki.getColNames().isEmpty()) { + if (primaryKeyInfo != null && !primaryKeyInfo.getColNames().isEmpty()) { ImmutableBitSet.Builder keys = ImmutableBitSet.builder(); - for (String pkColName : pki.getColNames().values()) { + for (String pkColName : primaryKeyInfo.getColNames().values()) { int pkPos; for (pkPos = 0; pkPos < rowType.getFieldNames().size(); pkPos++) { String colName = rowType.getFieldNames().get(pkPos); @@ -275,99 +276,89 @@ public boolean isKey(ImmutableBitSet columns) { nonNullbuilder.add(key); } // Then UKs - final UniqueConstraint uki; - try { - uki = Hive.get().getReliableUniqueConstraints( - hiveTblMetadata.getDbName(), hiveTblMetadata.getTableName()); - } catch (HiveException e) { - throw new RuntimeException(e); - } - for (List ukCols : uki.getUniqueConstraints().values()) { - ImmutableBitSet.Builder keys = ImmutableBitSet.builder(); - boolean isNonNullable = true; - for (UniqueConstraintCol ukCol : ukCols) { - int ukPos; - for (ukPos = 0; ukPos < rowType.getFieldNames().size(); ukPos++) { - String colName = rowType.getFieldNames().get(ukPos); - if (ukCol.colName.equals(colName)) { - if(rowType.getFieldList().get(ukPos).getType().isNullable()) { - // they should all be nullable - isNonNullable = false; + if (uniqueKeyInfo != null && !uniqueKeyInfo.getUniqueConstraints().isEmpty()) { + for (List ukCols : uniqueKeyInfo.getUniqueConstraints().values()) { + ImmutableBitSet.Builder keys = ImmutableBitSet.builder(); + boolean isNonNullable = true; + for (UniqueConstraintCol ukCol : ukCols) { + int ukPos; + for (ukPos = 0; ukPos < rowType.getFieldNames().size(); ukPos++) { + String colName = rowType.getFieldNames().get(ukPos); + if (ukCol.colName.equals(colName)) { + if (rowType.getFieldList().get(ukPos).getType().isNullable()) { + // they should all be nullable + isNonNullable = false; + } + break; } - break; } + if (ukPos == rowType.getFieldNames().size()) { + LOG.error("Column for unique constraint definition " + ukCol.colName + " not found"); + } + keys.set(ukPos); } - if (ukPos == rowType.getFieldNames().size()) { - LOG.error("Column for unique constraint definition " + ukCol.colName + " not found"); + ImmutableBitSet key = keys.build(); + builder.add(key); + if (isNonNullable) { + nonNullbuilder.add(key); } - keys.set(ukPos); - } - ImmutableBitSet key = keys.build(); - builder.add(key); - if(isNonNullable) { - nonNullbuilder.add(key); } } return new Pair<>(builder.build(), nonNullbuilder.build()); } private List generateReferentialConstraints() { - final ForeignKeyInfo fki; - try { - fki = Hive.get().getReliableForeignKeys( - hiveTblMetadata.getDbName(), hiveTblMetadata.getTableName()); - } catch (HiveException e) { - throw new RuntimeException(e); - } ImmutableList.Builder builder = ImmutableList.builder(); - for (List fkCols : fki.getForeignKeys().values()) { - List foreignKeyTableQualifiedName = qualifiedTblName; - String parentDatabaseName = fkCols.get(0).parentDatabaseName; - String parentTableName = fkCols.get(0).parentTableName; - List parentTableQualifiedName = new ArrayList<>(); - if (parentDatabaseName != null && !parentDatabaseName.isEmpty()) { - parentTableQualifiedName.add(parentDatabaseName); - } - parentTableQualifiedName.add(parentTableName); - Table parentTab = null; - try { - // TODO: We have a cache for Table objects in SemanticAnalyzer::getTableObjectByName() - // We need to move that cache elsewhere and use it from places like this. - parentTab = Hive.get().getTable(parentDatabaseName, parentTableName); - } catch (HiveException e) { - throw new RuntimeException(e); - } - if (parentTab == null) { - LOG.error("Table for primary key not found: " - + "databaseName: " + parentDatabaseName+ ", " - + "tableName: " + parentTableName); - return ImmutableList.of(); - } - ImmutableList.Builder keys = ImmutableList.builder(); - for (ForeignKeyCol fkCol : fkCols) { - int fkPos; - for (fkPos = 0; fkPos < rowType.getFieldNames().size(); fkPos++) { - String fkColName = rowType.getFieldNames().get(fkPos); - if (fkColName.equals(fkCol.childColName)) { - break; - } + if (foreignKeyInfo != null && !foreignKeyInfo.getForeignKeys().isEmpty()) { + for (List fkCols : foreignKeyInfo.getForeignKeys().values()) { + List foreignKeyTableQualifiedName = qualifiedTblName; + String parentDatabaseName = fkCols.get(0).parentDatabaseName; + String parentTableName = fkCols.get(0).parentTableName; + List parentTableQualifiedName = new ArrayList<>(); + if (parentDatabaseName != null && !parentDatabaseName.isEmpty()) { + parentTableQualifiedName.add(parentDatabaseName); } - int pkPos; - for (pkPos = 0; pkPos < parentTab.getAllCols().size(); pkPos++) { - String pkColName = parentTab.getAllCols().get(pkPos).getName(); - if (pkColName.equals(fkCol.parentColName)) { - break; - } + parentTableQualifiedName.add(parentTableName); + Table parentTab = null; + try { + // TODO: We have a cache for Table objects in SemanticAnalyzer::getTableObjectByName() + // We need to move that cache elsewhere and use it from places like this. + parentTab = Hive.get().getTable(parentDatabaseName, parentTableName); + } catch (HiveException e) { + throw new RuntimeException(e); } - if (fkPos == rowType.getFieldNames().size() - || pkPos == parentTab.getAllCols().size()) { - LOG.error("Column for foreign key definition " + fkCol + " not found"); + if (parentTab == null) { + LOG.error("Table for primary key not found: " + + "databaseName: " + parentDatabaseName + ", " + + "tableName: " + parentTableName); return ImmutableList.of(); } - keys.add(IntPair.of(fkPos, pkPos)); + ImmutableList.Builder keys = ImmutableList.builder(); + for (ForeignKeyCol fkCol : fkCols) { + int fkPos; + for (fkPos = 0; fkPos < rowType.getFieldNames().size(); fkPos++) { + String fkColName = rowType.getFieldNames().get(fkPos); + if (fkColName.equals(fkCol.childColName)) { + break; + } + } + int pkPos; + for (pkPos = 0; pkPos < parentTab.getAllCols().size(); pkPos++) { + String pkColName = parentTab.getAllCols().get(pkPos).getName(); + if (pkColName.equals(fkCol.parentColName)) { + break; + } + } + if (fkPos == rowType.getFieldNames().size() + || pkPos == parentTab.getAllCols().size()) { + LOG.error("Column for foreign key definition " + fkCol + " not found"); + return ImmutableList.of(); + } + keys.add(IntPair.of(fkPos, pkPos)); + } + builder.add(RelReferentialConstraintImpl.of(foreignKeyTableQualifiedName, + parentTableQualifiedName, keys.build())); } - builder.add(RelReferentialConstraintImpl.of(foreignKeyTableQualifiedName, - parentTableQualifiedName, keys.build())); } return builder.build(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 4011d99c92..a323854d70 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -135,11 +135,13 @@ import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.metadata.ForeignKeyInfo; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.NotNullConstraint; import org.apache.hadoop.hive.ql.metadata.PrimaryKeyInfo; import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.metadata.UniqueConstraint; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException; import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature; @@ -2884,6 +2886,12 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc // 2. Get Table Metadata Table tabMetaData = qb.getMetaData().getSrcForAlias(tableAlias); + PrimaryKeyInfo pki = db.getReliablePrimaryKeys( + tabMetaData.getDbName(), tabMetaData.getTableName()); + ForeignKeyInfo fki = db.getReliableForeignKeys( + tabMetaData.getDbName(), tabMetaData.getTableName()); + UniqueConstraint uki = db.getReliableUniqueConstraints( + tabMetaData.getDbName(), tabMetaData.getTableName()); // 3. Get Table Logical Schema (Row Type) // NOTE: Table logical schema = Non Partition Cols + Partition Cols + @@ -2995,7 +3003,7 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc dataSource, RelDataTypeImpl.proto(rowType), metrics, DruidTable.DEFAULT_TIMESTAMP_COLUMN, intervals, null, null); optTable = new RelOptHiveTable(relOptSchema, relOptSchema.getTypeFactory(), fullyQualifiedTabName, - rowType, tabMetaData, nonPartitionColumns, partitionColumns, virtualCols, conf, + rowType, tabMetaData, pki, fki, uki, nonPartitionColumns, partitionColumns, virtualCols, conf, partitionCache, colStatsCache, noColsMissingStats); final TableScan scan = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), optTable, null == tableAlias ? tabMetaData.getTableName() : tableAlias, @@ -3006,7 +3014,7 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc optTable, druidTable, ImmutableList.of(scan), DruidSqlOperatorConverter.getDefaultMap()); } else { optTable = new RelOptHiveTable(relOptSchema, relOptSchema.getTypeFactory(), fullyQualifiedTabName, - rowType, tabMetaData, nonPartitionColumns, partitionColumns, virtualCols, conf, + rowType, tabMetaData, pki, fki, uki, nonPartitionColumns, partitionColumns, virtualCols, conf, partitionCache, colStatsCache, noColsMissingStats); final HiveTableScan hts = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), optTable, @@ -3052,7 +3060,7 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc } fullyQualifiedTabName.add(tabMetaData.getTableName()); optTable = new RelOptHiveTable(relOptSchema, relOptSchema.getTypeFactory(), fullyQualifiedTabName, - rowType, tabMetaData, nonPartitionColumns, partitionColumns, virtualCols, conf, + rowType, tabMetaData, pki, fki, uki, nonPartitionColumns, partitionColumns, virtualCols, conf, partitionCache, colStatsCache, noColsMissingStats); // Build Hive Table Scan Rel tableRel = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), optTable, @@ -3084,10 +3092,10 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc private RelDataType inferNotNullableColumns(Table tabMetaData, RelDataType rowType) throws HiveException { // Retrieve not null constraints - final NotNullConstraint nnc = Hive.get().getReliableNotNullConstraints( + final NotNullConstraint nnc = db.getReliableNotNullConstraints( tabMetaData.getDbName(), tabMetaData.getTableName()); // Retrieve primary key constraints (cannot be null) - final PrimaryKeyInfo pkc = Hive.get().getReliablePrimaryKeys( + final PrimaryKeyInfo pkc = db.getReliablePrimaryKeys( tabMetaData.getDbName(), tabMetaData.getTableName()); if (nnc.getNotNullConstraints().isEmpty() && pkc.getColNames().isEmpty()) { return rowType;