diff --git itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java index 31d9b715def69a8e057258378b560e4dfcf0f17c..b747484415d3ea3a302a14d341d4cd1c5eaed335 100644 --- itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java +++ itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java @@ -320,10 +320,10 @@ public void updateCreationMetadata(String catName, String dbname, String tablena } @Override - public void alterTable(String catName, String dbName, String name, Table newTable) + public void alterTable(String catName, String dbName, String name, Table newTable, long queryTxnId, String queryValidWriteIds) throws InvalidObjectException, MetaException { if (shouldEventSucceed) { - objectStore.alterTable(catName, dbName, name, newTable); + objectStore.alterTable(catName, dbName, name, newTable, queryTxnId, queryValidWriteIds); } else { throw new RuntimeException("Event failed."); } @@ -385,9 +385,9 @@ public PartitionValuesResponse listPartitionValues(String catName, String db_nam @Override public void alterPartition(String catName, String dbName, String tblName, List partVals, - Partition newPart) throws InvalidObjectException, MetaException { + Partition newPart, long queryTxnId, String queryValidWriteIds) throws InvalidObjectException, MetaException { if (shouldEventSucceed) { - objectStore.alterPartition(catName, dbName, tblName, partVals, newPart); + objectStore.alterPartition(catName, dbName, tblName, partVals, newPart, queryTxnId, queryValidWriteIds); } else { throw new RuntimeException("Event failed."); } @@ -396,9 +396,9 @@ public void alterPartition(String catName, String dbName, String tblName, List> partValsList, List newParts, - long txnId, String writeIdList, long writeId) + long writeId, long queryTxnId, String queryValidWriteIds) throws InvalidObjectException, MetaException { - objectStore.alterPartitions(catName, dbName, tblName, partValsList, newParts, txnId, writeIdList, writeId); + objectStore.alterPartitions(catName, dbName, tblName, partValsList, newParts, writeId, queryTxnId, queryValidWriteIds); } @Override diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java index 49752e5d44a25c67abbd058598ae4cef8caa7334..61fb3d391414faad43990d9f73e92602dd77945d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java @@ -46,15 +46,11 @@ import org.apache.hadoop.hive.ql.DriverContext; import org.apache.hadoop.hive.ql.QueryPlan; import org.apache.hadoop.hive.ql.QueryState; -import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ColumnStatsUpdateWork; import org.apache.hadoop.hive.ql.plan.api.StageType; -import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 3b1e872e3b6073cdb8480d16e7dc1746c53088b8..71b9587697348ef9ba4c3d6ae101b0b2e6993562 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -1301,7 +1301,8 @@ private int alterMaterializedView(Hive db, AlterMaterializedViewDesc alterMVDesc throw new AssertionError("Unsupported alter materialized view type! : " + alterMVDesc.getOp()); } - db.alterTable(mv,environmentContext); + db.alterTable(mv, false, environmentContext, true); + return 0; } @@ -1451,7 +1452,7 @@ private int alterTableAlterPart(Hive db, AlterTableAlterPartDesc alterPartitionD tbl.getTTable().setPartitionKeys(newPartitionKeys); - db.alterTable(tbl, null); + db.alterTable(tbl, false, null, true); work.getInputs().add(new ReadEntity(tbl)); // We've already locked the table as the input, don't relock it as the output. @@ -1477,7 +1478,7 @@ private int touch(Hive db, AlterTableSimpleDesc touchDesc) environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE); if (touchDesc.getPartSpec() == null) { - db.alterTable(tbl, environmentContext); + db.alterTable(tbl, false, environmentContext, true); work.getInputs().add(new ReadEntity(tbl)); addIfAbsentByName(new WriteEntity(tbl, WriteEntity.WriteType.DDL_NO_LOCK)); } else { @@ -1486,7 +1487,7 @@ private int touch(Hive db, AlterTableSimpleDesc touchDesc) throw new HiveException("Specified partition does not exist"); } try { - db.alterPartition(touchDesc.getTableName(), part, environmentContext); + db.alterPartition(touchDesc.getTableName(), part, environmentContext, true); } catch (InvalidOperationException e) { throw new HiveException(e); } @@ -1835,7 +1836,7 @@ private int archive(Hive db, AlterTableSimpleDesc simpleDesc, authority.toString(), harPartitionDir.getPath()); // make in Path to ensure no slash at the end setArchived(p, harPath, partSpecInfo.values.size()); - db.alterPartition(simpleDesc.getTableName(), p, null); + db.alterPartition(simpleDesc.getTableName(), p, null, true); } } catch (Exception e) { throw new HiveException("Unable to change the partition info for HAR", e); @@ -2041,7 +2042,7 @@ private int unarchive(Hive db, AlterTableSimpleDesc simpleDesc) for(Partition p: partitions) { setUnArchived(p); try { - db.alterPartition(simpleDesc.getTableName(), p, null); + db.alterPartition(simpleDesc.getTableName(), p, null, true); } catch (InvalidOperationException e) { throw new HiveException(e); } @@ -3730,7 +3731,8 @@ private int describeTable(Hive db, DescTableDesc descTbl) throws HiveException, } else { cols = Hive.getFieldsFromDeserializer(colPath, deserializer); List parts = db.getPartitionNames(dbTab[0].toLowerCase(), dbTab[1].toLowerCase(), (short) -1); - AggrStats aggrStats = db.getAggrColStatsFor(dbTab[0].toLowerCase(), dbTab[1].toLowerCase(), colNames, parts); + AggrStats aggrStats = db.getAggrColStatsFor( + dbTab[0].toLowerCase(), dbTab[1].toLowerCase(), colNames, parts, false); colStats = aggrStats.getColStats(); if (parts.size() == aggrStats.getPartsFound()) { StatsSetupConst.setColumnStatsState(tblProps, colNames); @@ -3741,13 +3743,15 @@ private int describeTable(Hive db, DescTableDesc descTbl) throws HiveException, tbl.setParameters(tblProps); } else { cols = Hive.getFieldsFromDeserializer(colPath, deserializer); - colStats = db.getTableColumnStatistics(dbTab[0].toLowerCase(), dbTab[1].toLowerCase(), colNames); + colStats = db.getTableColumnStatistics( + dbTab[0].toLowerCase(), dbTab[1].toLowerCase(), colNames, false); } } else { List partitions = new ArrayList(); partitions.add(part.getName()); cols = Hive.getFieldsFromDeserializer(colPath, deserializer); - colStats = db.getPartitionColumnStatistics(dbTab[0].toLowerCase(), dbTab[1].toLowerCase(), partitions, colNames).get(part.getName()); + colStats = db.getPartitionColumnStatistics(dbTab[0].toLowerCase(), + dbTab[1].toLowerCase(), partitions, colNames, false).get(part.getName()); } } else { cols = Hive.getFieldsFromDeserializer(colPath, deserializer); @@ -3965,10 +3969,14 @@ private int alterTable(Hive db, AlterTableDesc alterTbl) throws HiveException { environmentContext = new EnvironmentContext(); } environmentContext.putToProperties(HiveMetaHook.ALTER_TABLE_OPERATION_TYPE, alterTbl.getOp().name()); + // Note: in the old default overloads that I've removed, "transactional" was true for tables, + // but false for partitions. Seems to be ok here because we are not updating + // partition-stats-related stuff from this call (alterTable). if (allPartitions == null) { - db.alterTable(alterTbl.getOldName(), tbl, alterTbl.getIsCascade(), environmentContext); + db.alterTable(alterTbl.getOldName(), tbl, alterTbl.getIsCascade(), environmentContext, true); } else { - db.alterPartitions(Warehouse.getQualifiedName(tbl.getTTable()), allPartitions, environmentContext); + db.alterPartitions( + Warehouse.getQualifiedName(tbl.getTTable()), allPartitions, environmentContext, false); } // Add constraints if necessary addConstraints(db, alterTbl); @@ -4943,7 +4951,7 @@ private int createTable(Hive db, CreateTableDesc crtTbl) throws HiveException { // create the table if (crtTbl.getReplaceMode()) { // replace-mode creates are really alters using CreateTableDesc. - db.alterTable(tbl, null); + db.alterTable(tbl, false, null, true); } else { if ((foreignKeys != null && foreignKeys.size() > 0) || (primaryKeys != null && primaryKeys.size() > 0) || @@ -5173,7 +5181,7 @@ private int createView(Hive db, CreateViewDesc crtView) throws HiveException { oldview.setOutputFormatClass(crtView.getOutputFormat()); } oldview.checkValidity(null); - db.alterTable(crtView.getViewName(), oldview, null); + db.alterTable(crtView.getViewName(), oldview, false, null, true); addIfAbsentByName(new WriteEntity(oldview, WriteEntity.WriteType.DDL_NO_LOCK)); } else { // We create new view diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java index 19097f5e70d7ff631764e11a3b4b851230dc4e06..a2d634084c8837652b0cad239a23afbdaa0928db 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java @@ -793,7 +793,7 @@ private void updatePartitionBucketSortColumns(Hive db, Table table, Partition pa } if (updateBucketCols || updateSortCols) { - db.alterPartition(table.getDbName(), table.getTableName(), partn, null); + db.alterPartition(table.getDbName(), table.getTableName(), partn, null, true); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/hooks/UpdateInputAccessTimeHook.java ql/src/java/org/apache/hadoop/hive/ql/hooks/UpdateInputAccessTimeHook.java index fc56a8be3b2607f8a0c196e81f35ad4128c13794..4cf7c25a8260e86869d35cfaf97aacfece988e1f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/hooks/UpdateInputAccessTimeHook.java +++ ql/src/java/org/apache/hadoop/hive/ql/hooks/UpdateInputAccessTimeHook.java @@ -63,7 +63,7 @@ public void run(HookContext hookContext) throws Exception { String tblName = re.getTable().getTableName(); Table t = db.getTable(dbName, tblName); t.setLastAccessTime(lastAccessTime); - db.alterTable(dbName + "." + tblName, t, null); + db.alterTable(dbName + "." + tblName, t, false, null, true); break; } case PARTITION: { @@ -73,9 +73,9 @@ public void run(HookContext hookContext) throws Exception { Table t = db.getTable(dbName, tblName); p = db.getPartition(t, p.getSpec(), false); p.setLastAccessTime(lastAccessTime); - db.alterPartition(dbName, tblName, p, null); + db.alterPartition(dbName, tblName, p, null, true); t.setLastAccessTime(lastAccessTime); - db.alterTable(dbName + "." + tblName, t, null); + db.alterTable(dbName + "." + tblName, t, false, null, true); break; } default: diff --git ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java index f126550fea6d90e47f0210f6607bab666b85b331..f8da7a7613d1af8a5a7546a56772d55c4042569c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java @@ -1692,11 +1692,16 @@ public static TableSnapshot getTableSnapshot( if (txnId > 0 && isTransactionalTable(tbl)) { validWriteIdList = getTableValidWriteIdList(conf, fullTableName); if (isStatsUpdater) { - // TODO# it should be invalid to update stats without write ID... - // Why would there be a stats updater that doesn't have a write ID? writeId = SessionState.get().getTxnMgr() != null ? SessionState.get().getTxnMgr().getAllocatedTableWriteId( tbl.getDbName(), tbl.getTableName()) : -1; + if (writeId < 1) { + // TODO: it should be invalid to update txn stats without write ID... + // Why would there be a stats updater that doesn't have a write ID? + // The end result will be that stats will be set to invalid state right now. + LOG.warn("Stats updater for {}.{} doesn't have a write ID", + tbl.getDbName(), tbl.getTableName()); + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java index 18eed52ca13ff849080996305815eea2d12f6673..122712609571f9e71babd10cb7be940541b63324 100644 --- ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java +++ ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java @@ -1041,6 +1041,7 @@ private long getTableWriteId( } try { long writeId = getMS().allocateTableWriteId(txnId, dbName, tableName); + LOG.debug("Allocated write ID {} for {}.{}", writeId, dbName, tableName); tableWriteIds.put(fullTableName, writeId); return writeId; } catch (TException e) { diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 9511d63430050a7cf7a5c4f1518d0217aad1a739..1434125de0632e2677a255612d3417006d583dc8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -581,15 +581,11 @@ public void createTable(String tableName, List columns, List par createTable(tbl); } - public void alterTable(Table newTbl, EnvironmentContext environmentContext) - throws HiveException { - alterTable(newTbl.getDbName(), newTbl.getTableName(), newTbl, false, environmentContext); - } - - public void alterTable(String fullyQlfdTblName, Table newTbl, EnvironmentContext environmentContext) - throws HiveException { - alterTable(fullyQlfdTblName, newTbl, false, environmentContext); + public void alterTable(Table newTbl, boolean cascade, EnvironmentContext environmentContext, + boolean transactional) throws HiveException { + alterTable(newTbl.getDbName(), + newTbl.getTableName(), newTbl, cascade, environmentContext, transactional); } /** @@ -612,16 +608,13 @@ public void alterTable(String fullyQlfdTblName, Table newTbl, EnvironmentContext alterTable(names[0], names[1], newTbl, false, environmentContext, transactional); } - public void alterTable(String fullyQlfdTblName, Table newTbl, boolean cascade, EnvironmentContext environmentContext) + public void alterTable(String fullyQlfdTblName, Table newTbl, boolean cascade, + EnvironmentContext environmentContext, boolean transactional) throws HiveException { String[] names = Utilities.getDbTableName(fullyQlfdTblName); - alterTable(names[0], names[1], newTbl, cascade, environmentContext); - } - public void alterTable(String dbName, String tblName, Table newTbl, boolean cascade, - EnvironmentContext environmentContext) - throws HiveException { - alterTable(dbName, tblName, newTbl, cascade, environmentContext, true); + alterTable(names[0], names[1], newTbl, cascade, environmentContext, transactional); } + public void alterTable(String dbName, String tblName, Table newTbl, boolean cascade, EnvironmentContext environmentContext, boolean transactional) throws HiveException { @@ -641,10 +634,11 @@ public void alterTable(String dbName, String tblName, Table newTbl, boolean casc // Take a table snapshot and set it to newTbl. if (transactional) { - setTableSnapshotForTransactionalTable(conf, newTbl, true); + setTableSnapshotForTransactionalTable(environmentContext, conf, newTbl, true); } - getMSC().alter_table_with_environmentContext(dbName, tblName, newTbl.getTTable(), environmentContext); + getMSC().alter_table_with_environmentContext( + dbName, tblName, newTbl.getTTable(), environmentContext); } catch (MetaException e) { throw new HiveException("Unable to alter table. " + e.getMessage(), e); } catch (TException e) { @@ -672,28 +666,11 @@ public void updateCreationMetadata(String dbName, String tableName, CreationMeta * if the changes in metadata is not acceptable * @throws TException */ - public void alterPartition(String tblName, Partition newPart, EnvironmentContext environmentContext) + public void alterPartition(String tblName, Partition newPart, + EnvironmentContext environmentContext, boolean transactional) throws InvalidOperationException, HiveException { String[] names = Utilities.getDbTableName(tblName); - alterPartition(names[0], names[1], newPart, environmentContext); - } - - /** - * Updates the existing partition metadata with the new metadata. - * - * @param dbName - * name of the exiting table's database - * @param tblName - * name of the existing table - * @param newPart - * new partition - * @throws InvalidOperationException - * if the changes in metadata is not acceptable - * @throws TException - */ - public void alterPartition(String dbName, String tblName, Partition newPart, EnvironmentContext environmentContext) - throws InvalidOperationException, HiveException { - alterPartition(dbName, tblName, newPart, environmentContext, true); + alterPartition(names[0], names[1], newPart, environmentContext, transactional); } /** @@ -723,10 +700,14 @@ public void alterPartition(String dbName, String tblName, Partition newPart, location = Utilities.getQualifiedPath(conf, new Path(location)); newPart.setLocation(location); } + if (environmentContext == null) { + environmentContext = new EnvironmentContext(); + } if (transactional) { - setTableSnapshotForTransactionalPartition(conf, newPart, true); + setTableSnapshotForTransactionalPartition(environmentContext, conf, newPart, true); } - getSynchronizedMSC().alter_partition(dbName, tblName, newPart.getTPartition(), environmentContext); + getSynchronizedMSC().alter_partition( + dbName, tblName, newPart.getTPartition(), environmentContext); } catch (MetaException e) { throw new HiveException("Unable to alter partition. " + e.getMessage(), e); @@ -743,10 +724,6 @@ private void validatePartition(Partition newPart) throws HiveException { newPart.checkValidity(); } - public void alterPartitions(String tblName, List newParts, EnvironmentContext environmentContext) - throws InvalidOperationException, HiveException { - alterPartitions(tblName, newParts, environmentContext, false); - } /** * Updates the existing table metadata with the new metadata. * @@ -918,7 +895,7 @@ public void createTable(Table tbl, boolean ifNotExists, } } // Set table snapshot to api.Table to make it persistent. - setTableSnapshotForTransactionalTable(conf, tbl, true); + setTableSnapshotForTransactionalTable(null, conf, tbl, true); if (primaryKeys == null && foreignKeys == null && uniqueConstraints == null && notNullConstraints == null && defaultConstraints == null && checkConstraints == null) { @@ -1150,6 +1127,7 @@ public Table getTable(final String dbName, final String tableName, // Get the table from metastore org.apache.hadoop.hive.metastore.api.Table tTable = null; try { + // Note: this is currently called w/true from StatsOptimizer only. if (checkTransactional) { ValidWriteIdList validWriteIdList = null; long txnId = SessionState.get().getTxnMgr() != null ? @@ -1814,7 +1792,8 @@ public Partition loadPartition(Path loadPath, Table tbl, Map par Partition newTPart = oldPart != null ? oldPart : new Partition(tbl, partSpec, newPartPath); alterPartitionSpecInMemory(tbl, partSpec, newTPart.getTPartition(), inheritTableSpecs, newPartPath.toString()); validatePartition(newTPart); - setTableSnapshotForTransactionalPartition(conf, newTPart, true); + EnvironmentContext ec = new EnvironmentContext(); + setTableSnapshotForTransactionalPartition(ec, conf, newTPart, true); // If config is set, table is not temporary and partition being inserted exists, capture // the list of files added. For not yet existing partitions (insert overwrite to new partition @@ -1880,7 +1859,7 @@ public Partition loadPartition(Path loadPath, Table tbl, Map par // insert into table T partition (ds) values ('Joe', 'today'); -- will fail with AlreadyExistsException // In that case, we want to retry with alterPartition. LOG.debug("Caught AlreadyExistsException, trying to alter partition instead"); - setStatsPropAndAlterPartition(hasFollowingStatsTask, tbl, newTPart); + setStatsPropAndAlterPartition(hasFollowingStatsTask, tbl, newTPart, ec); } catch (Exception e) { try { final FileSystem newPathFileSystem = newPartPath.getFileSystem(this.getConf()); @@ -1893,7 +1872,7 @@ public Partition loadPartition(Path loadPath, Table tbl, Map par throw e; } } else { - setStatsPropAndAlterPartition(hasFollowingStatsTask, tbl, newTPart); + setStatsPropAndAlterPartition(hasFollowingStatsTask, tbl, newTPart, ec); } perfLogger.PerfLogEnd("MoveTask", PerfLogger.LOAD_PARTITION); @@ -1993,15 +1972,13 @@ public boolean accept(Path path) { } private void setStatsPropAndAlterPartition(boolean hasFollowingStatsTask, Table tbl, - Partition newTPart) throws MetaException, TException { - EnvironmentContext environmentContext = null; + Partition newTPart, EnvironmentContext ec) throws MetaException, TException { if (hasFollowingStatsTask) { - environmentContext = new EnvironmentContext(); - environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE); + ec.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE); } LOG.debug("Altering existing partition " + newTPart.getSpec()); getSynchronizedMSC().alter_partition(tbl.getDbName(), tbl.getTableName(), - newTPart.getTPartition(), environmentContext); + newTPart.getTPartition(), ec); } /** @@ -2428,7 +2405,7 @@ public void loadTable(Path loadPath, String tableName, LoadFileType loadFileType environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE); } - alterTable(tbl, environmentContext); + alterTable(tbl, false, environmentContext, true); if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary()) { fireInsertEvent(tbl, null, (loadFileType == LoadFileType.REPLACE_ALL), newFiles); @@ -2456,8 +2433,7 @@ public Partition createPartition(Table tbl, Map partSpec) throws org.apache.hadoop.hive.metastore.api.Partition part = Partition.createMetaPartitionObject(tbl, partSpec, null); AcidUtils.TableSnapshot tableSnapshot = AcidUtils.getTableSnapshot(conf, tbl); - part.setTxnId(tableSnapshot != null ? tableSnapshot.getTxnId() : 0); - part.setValidWriteIdList(tableSnapshot != null ? tableSnapshot.getValidWriteIdList() : null); + part.setWriteId(tableSnapshot != null ? tableSnapshot.getWriteId() : 0); return new Partition(tbl, getMSC().add_partition(part)); } catch (Exception e) { LOG.error(StringUtils.stringifyException(e)); @@ -2475,9 +2451,8 @@ public Partition createPartition(Table tbl, Map partSpec) throws for (int i = 0; i < size; ++i) { org.apache.hadoop.hive.metastore.api.Partition tmpPart = convertAddSpecToMetaPartition(tbl, addPartitionDesc.getPartition(i), conf); - if (tmpPart != null && tableSnapshot != null && tableSnapshot.getTxnId() > 0) { - tmpPart.setTxnId(tableSnapshot.getTxnId()); - tmpPart.setValidWriteIdList(tableSnapshot.getValidWriteIdList()); + if (tmpPart != null && tableSnapshot != null && tableSnapshot.getWriteId() > 0) { + tmpPart.setWriteId(tableSnapshot.getWriteId()); } in.add(tmpPart); } @@ -2675,8 +2650,7 @@ private void alterPartitionSpec(Table tbl, if (!org.apache.commons.lang.StringUtils.isEmpty(tbl.getDbName())) { fullName = tbl.getFullyQualifiedName(); } - Partition newPart = new Partition(tbl, tpart); - alterPartition(fullName, newPart, null); + alterPartition(fullName, new Partition(tbl, tpart), null, true); } private void alterPartitionSpecInMemory(Table tbl, @@ -4461,11 +4435,6 @@ public boolean setPartitionColumnStatistics( } public List getTableColumnStatistics( - String dbName, String tableName, List colNames) throws HiveException { - return getTableColumnStatistics(dbName, tableName, colNames, false); - } - - public List getTableColumnStatistics( String dbName, String tableName, List colNames, boolean checkTransactional) throws HiveException { @@ -4489,11 +4458,6 @@ public boolean setPartitionColumnStatistics( } } - public Map> getPartitionColumnStatistics(String dbName, - String tableName, List partNames, List colNames) throws HiveException { - return getPartitionColumnStatistics(dbName, tableName, partNames, colNames, false); - } - public Map> getPartitionColumnStatistics( String dbName, String tableName, List partNames, List colNames, boolean checkTransactional) @@ -4517,11 +4481,6 @@ public boolean setPartitionColumnStatistics( } public AggrStats getAggrColStatsFor(String dbName, String tblName, - List colNames, List partName) { - return getAggrColStatsFor(dbName, tblName, colNames, partName, false); - } - - public AggrStats getAggrColStatsFor(String dbName, String tblName, List colNames, List partName, boolean checkTransactional) { long txnId = -1; String writeIdList = null; @@ -5332,34 +5291,42 @@ public StorageHandlerInfo getStorageHandlerInfo(Table table) } } - private void setTableSnapshotForTransactionalTable( - HiveConf conf, Table newTbl, boolean isStatsUpdater) - throws LockException { + private void setTableSnapshotForTransactionalTable(EnvironmentContext ec, HiveConf conf, + Table newTbl, boolean isStatsUpdater) throws LockException { org.apache.hadoop.hive.metastore.api.Table newTTbl = newTbl.getTTable(); AcidUtils.TableSnapshot tableSnapshot = AcidUtils.getTableSnapshot(conf, newTbl, isStatsUpdater); + if (tableSnapshot == null) return; + if (ec != null) { // Can be null for create table case; we don't need to verify txn stats. + ec.putToProperties(StatsSetupConst.TXN_ID, Long.toString(tableSnapshot.getTxnId())); + if (tableSnapshot.getValidWriteIdList() != null) { + ec.putToProperties(StatsSetupConst.VALID_WRITE_IDS, tableSnapshot.getValidWriteIdList()); + } else { + LOG.warn("Table snapshot has null write IDs for " + newTbl); + } + } - newTTbl.setTxnId(tableSnapshot != null ? tableSnapshot.getTxnId() : -1); - newTTbl.setValidWriteIdList( - tableSnapshot != null ? tableSnapshot.getValidWriteIdList() : null); if (isStatsUpdater) { - newTTbl.setWriteId(tableSnapshot != null ? tableSnapshot.getWriteId() : -1); + newTTbl.setWriteId(tableSnapshot.getWriteId()); } } - private void setTableSnapshotForTransactionalPartition( - HiveConf conf, Partition partition, boolean isStatsUpdater) - throws LockException { - + private void setTableSnapshotForTransactionalPartition(EnvironmentContext ec, HiveConf conf, + Partition partition, boolean isStatsUpdater) throws LockException { AcidUtils.TableSnapshot tableSnapshot = AcidUtils.getTableSnapshot(conf, partition.getTable(), isStatsUpdater); org.apache.hadoop.hive.metastore.api.Partition tpartition = partition.getTPartition(); - tpartition.setTxnId(tableSnapshot != null ? tableSnapshot.getTxnId() : -1); - tpartition.setValidWriteIdList( - tableSnapshot != null ? tableSnapshot.getValidWriteIdList() : null); + if (tableSnapshot == null) return; + ec.putToProperties(StatsSetupConst.TXN_ID, Long.toString(tableSnapshot.getTxnId())); + if (tableSnapshot.getValidWriteIdList() != null) { + ec.putToProperties(StatsSetupConst.VALID_WRITE_IDS, tableSnapshot.getValidWriteIdList()); + } else { + LOG.warn("Table snapshot has null write IDs for " + partition); + } + if (isStatsUpdater) { - tpartition.setWriteId(tableSnapshot != null ? tableSnapshot.getWriteId() : -1); + tpartition.setWriteId(tableSnapshot.getWriteId()); } } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java index 6cc6d02b147194ba15d3ae0dadaecb209b52c809..f66f47a838f874f96fe4482a993ca69a257cdc74 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java @@ -593,6 +593,7 @@ private void updateColStats(Set projIndxLst, boolean allowMissingStats) return getColStat(projIndxLst, HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_STATS_ESTIMATE_STATS)); } + /** Note: DOES NOT CHECK txn stats. */ public List getColStat(List projIndxLst, boolean allowMissingStats) { List colStatsBldr = Lists.newArrayList(); Set projIndxSet = new HashSet(projIndxLst); diff --git ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java index acebf520d185b0449143ef093f8453f240c63f7a..da4ef4a3adf47d319840211a102133637ee4a6d1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java +++ ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.FetchOperator; import org.apache.hadoop.hive.ql.io.AcidUtils; +import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; @@ -178,10 +179,13 @@ public int persistColumnStats(Hive db, Table tbl) throws HiveException, MetaExce } SetPartitionsStatsRequest request = new SetPartitionsStatsRequest(colStats); request.setNeedMerge(colStatDesc.isNeedMerge()); - if (AcidUtils.isTransactionalTable(tbl) && SessionState.get().getTxnMgr() != null) { - request.setTxnId(SessionState.get().getTxnMgr().getCurrentTxnId()); + HiveTxnManager txnMgr = AcidUtils.isTransactionalTable(tbl) + ? SessionState.get().getTxnMgr() : null; + if (txnMgr != null) { + request.setTxnId(txnMgr.getCurrentTxnId()); request.setValidWriteIdList(AcidUtils.getTableValidWriteIdList(conf, AcidUtils.getFullTableName(tbl.getDbName(), tbl.getTableName())).toString()); + request.setWriteId(txnMgr.getTableWriteId(tbl.getDbName(), tbl.getTableName())); } db.setPartitionColumnStatistics(request); return 0; diff --git ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUpdaterThread.java ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUpdaterThread.java index ddca70497a3f51c3ec9ea532fac2a42aa36149b3..bb181a192ac39c590c79b3c38aa1515cf16593ef 100644 --- ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUpdaterThread.java +++ ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUpdaterThread.java @@ -409,6 +409,8 @@ private String buildPartColStr(Table table) { List allCols) throws MetaException { ColumnStatistics existingStats = null; try { + // Note: this should NOT do txn verification - we want to get outdated stats, to + // see if we need to update anything. existingStats = rs.getTableColumnStatistics(cat, db, tbl, allCols); } catch (NoSuchObjectException e) { LOG.error("Cannot retrieve existing stats, skipping " + fullTableName, e); diff --git ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java index 494939a7994ea35ed25aea8648a6afb2ead3a248..cb6913e13182a2426af0a869da5805686d43942e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java @@ -128,6 +128,7 @@ /** * Collect table, partition and column level statistics + * Note: DOES NOT CHECK txn stats. * @param conf * - hive configuration * @param partList @@ -226,6 +227,7 @@ private static void estimateStatsForMissingCols(List neededColumns, List } } + /** Note: DOES NOT CHECK txn stats. */ public static Statistics collectStatistics(HiveConf conf, PrunedPartitionList partList, Table table, List schema, List neededColumns, ColumnStatsList colStatsCache, List referencedColumns, boolean fetchColStats) @@ -262,7 +264,10 @@ private static Statistics collectStatistics(HiveConf conf, PrunedPartitionList p List colStats = Lists.newArrayList(); if (fetchColStats) { - colStats = getTableColumnStats(table, schema, neededColumns, colStatsCache); + // Note: this is currently called from two notable places (w/false for checkTxn) + // 1) StatsRulesProcFactory.TableScanStatsRule via collectStatistics + // 2) RelOptHiveTable via getColStats and updateColStats. + colStats = getTableColumnStats(table, schema, neededColumns, colStatsCache, false); if(colStats == null) { colStats = Lists.newArrayList(); } @@ -378,8 +383,11 @@ private static Statistics collectStatistics(HiveConf conf, PrunedPartitionList p // size is 0, aggrStats is null after several retries. Thus, we can // skip the step to connect to the metastore. if (neededColsToRetrieve.size() > 0 && partNames.size() > 0) { + // Note: this is currently called from two notable places (w/false for checkTxn) + // 1) StatsRulesProcFactory.TableScanStatsRule via collectStatistics + // 2) RelOptHiveTable via getColStats and updateColStats. aggrStats = Hive.get().getAggrColStatsFor(table.getDbName(), table.getTableName(), - neededColsToRetrieve, partNames); + neededColsToRetrieve, partNames, false); } boolean statsRetrieved = aggrStats != null && @@ -990,7 +998,7 @@ else if(colTypeLowerCase.equals(serdeConstants.SMALLINT_TYPE_NAME)){ */ public static List getTableColumnStats( Table table, List schema, List neededColumns, - ColumnStatsList colStatsCache) { + ColumnStatsList colStatsCache, boolean checkTransactional) { if (table.isMaterializedTable()) { LOG.debug("Materialized table does not contain table statistics"); return null; @@ -1019,7 +1027,7 @@ else if(colTypeLowerCase.equals(serdeConstants.SMALLINT_TYPE_NAME)){ List stats = null; try { List colStat = Hive.get().getTableColumnStatistics( - dbName, tabName, colStatsToRetrieve); + dbName, tabName, colStatsToRetrieve, checkTransactional); stats = convertColStats(colStat, tabName); } catch (HiveException e) { LOG.error("Failed to retrieve table statistics: ", e); diff --git ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java index 930282d73ef4296c01d9cbabbe6eb753a022d731..d30bbde07172b20c86f075a962ed6e4764a9e08e 100755 --- ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java +++ ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java @@ -325,11 +325,8 @@ private void validateTable(Table tbl, String tableName) throws MetaException { tbl.getParameters().put(hive_metastoreConstants.DDL_TIME, ft.getParameters().get(hive_metastoreConstants.DDL_TIME)); // Txn stuff set by metastore - if (tbl.getTTable().isSetTxnId()) { - ft.getTTable().setTxnId(tbl.getTTable().getTxnId()); - } - if (tbl.getTTable().isSetValidWriteIdList()) { - ft.getTTable().setValidWriteIdList(tbl.getTTable().getValidWriteIdList()); + if (tbl.getTTable().isSetWriteId()) { + ft.getTTable().setWriteId(tbl.getTTable().getWriteId()); } assertTrue("Tables doesn't match: " + tableName + " (" + ft.getTTable() + "; " + tbl.getTTable() + ")", ft.getTTable().equals(tbl.getTTable())); @@ -600,7 +597,7 @@ public void testAutoPurgeTablesAndPartitions() throws Throwable { Table table = createPartitionedTable(dbName, tableName); table.getParameters().put("auto.purge", "true"); - hm.alterTable(tableName, table, null); + hm.alterTable(tableName, table, false, null, true); Map partitionSpec = new ImmutableMap.Builder() .put("ds", "20141216") diff --git ql/src/test/results/clientpositive/acid_stats.q.out ql/src/test/results/clientpositive/acid_stats.q.out index 969433ea24e19bed1e361f8190422d9bf0e8baba..fd4ebe75d40b849184aa00cb7536fef684cbad0c 100644 --- ql/src/test/results/clientpositive/acid_stats.q.out +++ ql/src/test/results/clientpositive/acid_stats.q.out @@ -45,47 +45,12 @@ PREHOOK: type: QUERY POSTHOOK: query: explain select count(key) from stats_part POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: stats_part - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL - Group By Operator - aggregations: count(key) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-0 Fetch Operator - limit: -1 + limit: 1 Processor Tree: ListSink @@ -102,47 +67,12 @@ PREHOOK: type: QUERY POSTHOOK: query: explain select count(key) from stats_part POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: stats_part - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(key) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-0 Fetch Operator - limit: -1 + limit: 1 Processor Tree: ListSink diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java standalone-metastore/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java index 78ea01d9687fe043d63441430c46b30c25cd9756..a13b40dcd4a743f71cc9ca51f67ff27ec5a388b5 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java @@ -149,6 +149,10 @@ public String getAggregator(Configuration conf) { public static final String CASCADE = "CASCADE"; + // TODO: when alter calls are switched to req/resp models, replace these and the above with fields. + public static final String TXN_ID = "WRITER_TXN_ID"; + public static final String VALID_WRITE_IDS = "WRITER_WRITE_ID"; + public static final String TRUE = "true"; public static final String FALSE = "false"; diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index 4e771975e1674635cb5a6aa8a4523884c6290d5f..8b2a6babeb117e150a2ad51e40d48adc506731de 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -150,7 +150,8 @@ public void alterTable(RawStore msdb, Warehouse wh, String catName, String dbnam msdb.openTransaction(); // get old table - oldt = msdb.getTable(catName, dbname, name, -1, null); + // Note: we don't verify stats here; it's done below in alterTableUpdateTableColumnStats. + oldt = msdb.getTable(catName, dbname, name, -1, null); if (oldt == null) { throw new InvalidOperationException("table " + TableName.getQualified(catName, dbname, name) + " doesn't exist"); @@ -275,7 +276,8 @@ public void alterTable(RawStore msdb, Warehouse wh, String catName, String dbnam columnStatsNeedUpdated.put(part, colStats); } } - msdb.alterTable(catName, dbname, name, newt); + // Do not verify stats parameters on a partitioned table. + msdb.alterTable(catName, dbname, name, newt, -1, null); // alterPartition is only for changing the partition location in the table rename if (dataWasMoved) { @@ -293,8 +295,8 @@ public void alterTable(RawStore msdb, Warehouse wh, String catName, String dbnam for (Partition part : partBatch) { partValues.add(part.getValues()); } - msdb.alterPartitions( - catName, newDbName, newTblName, partValues, partBatch, -1, null, -1); + msdb.alterPartitions(catName, newDbName, newTblName, partValues, + partBatch, -1, -1, null); } } @@ -305,7 +307,7 @@ public void alterTable(RawStore msdb, Warehouse wh, String catName, String dbnam msdb.updatePartitionColumnStatistics(newPartColStats, partColStats.getKey().getValues()); } } else { - alterTableUpdateTableColumnStats(msdb, oldt, newt); + alterTableUpdateTableColumnStats(msdb, oldt, newt, environmentContext); } } else { // operations other than table rename @@ -328,21 +330,23 @@ public void alterTable(RawStore msdb, Warehouse wh, String catName, String dbnam ColumnStatistics colStats = updateOrGetPartitionColumnStats(msdb, catName, dbname, name, part.getValues(), oldCols, oldt, part, null); assert(colStats == null); + // Note: we don't do txn stats validation here; this can only delete stats? if (cascade) { - msdb.alterPartition(catName, dbname, name, part.getValues(), part); + msdb.alterPartition(catName, dbname, name, part.getValues(), part, -1, null); } else { // update changed properties (stats) oldPart.setParameters(part.getParameters()); - msdb.alterPartition(catName, dbname, name, part.getValues(), oldPart); + msdb.alterPartition(catName, dbname, name, part.getValues(), oldPart, -1, null); } } - msdb.alterTable(catName, dbname, name, newt); + // Don't validate table-level stats for a partitoned table. + msdb.alterTable(catName, dbname, name, newt, -1, null); } else { LOG.warn("Alter table not cascaded to partitions."); - alterTableUpdateTableColumnStats(msdb, oldt, newt); + alterTableUpdateTableColumnStats(msdb, oldt, newt, environmentContext); } } else { - alterTableUpdateTableColumnStats(msdb, oldt, newt); + alterTableUpdateTableColumnStats(msdb, oldt, newt, environmentContext); } } @@ -445,6 +449,13 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String new_part.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(System .currentTimeMillis() / 1000)); } + long txnId = -1; + String validWriteIds = null; + if (environmentContext != null && environmentContext.isSetProperties() + && environmentContext.getProperties().containsKey(StatsSetupConst.VALID_WRITE_IDS)) { + txnId = Long.parseLong(environmentContext.getProperties().get(StatsSetupConst.TXN_ID)); + validWriteIds = environmentContext.getProperties().get(StatsSetupConst.VALID_WRITE_IDS); + } //alter partition if (part_vals == null || part_vals.size() == 0) { @@ -472,7 +483,8 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String updateOrGetPartitionColumnStats(msdb, catName, dbname, name, new_part.getValues(), oldPart.getSd().getCols(), tbl, new_part, null); } - msdb.alterPartition(catName, dbname, name, new_part.getValues(), new_part); + msdb.alterPartition( + catName, dbname, name, new_part.getValues(), new_part, txnId, validWriteIds); if (transactionalListeners != null && !transactionalListeners.isEmpty()) { MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventMessage.EventType.ALTER_PARTITION, @@ -607,7 +619,7 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String String newPartName = Warehouse.makePartName(tbl.getPartitionKeys(), new_part.getValues()); ColumnStatistics cs = updateOrGetPartitionColumnStats(msdb, catName, dbname, name, oldPart.getValues(), oldPart.getSd().getCols(), tbl, new_part, null); - msdb.alterPartition(catName, dbname, name, part_vals, new_part); + msdb.alterPartition(catName, dbname, name, part_vals, new_part, txnId, validWriteIds); if (cs != null) { cs.getStatsDesc().setPartName(newPartName); try { @@ -650,6 +662,7 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String return oldPart; } + @Deprecated @Override public List alterPartitions(final RawStore msdb, Warehouse wh, final String dbname, final String name, final List new_parts, @@ -678,6 +691,7 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String try { msdb.openTransaction(); + // Note: should we pass in write ID here? We only update stats on parts so probably not. Table tbl = msdb.getTable(catName, dbname, name, -1, null); if (tbl == null) { throw new InvalidObjectException( @@ -713,7 +727,7 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String } } - msdb.alterPartitions(catName, dbname, name, partValsList, new_parts, txnId, writeIdList, writeId); + msdb.alterPartitions(catName, dbname, name, partValsList, new_parts, writeId, txnId, writeIdList); Iterator oldPartsIt = oldParts.iterator(); for (Partition newPart : new_parts) { Partition oldPart; @@ -781,7 +795,8 @@ private Path constructRenamedPath(Path defaultNewPath, Path currentPath) { } @VisibleForTesting - void alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTable) + void alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTable, + EnvironmentContext ec) throws MetaException, InvalidObjectException { String catName = normalizeIdentifier(oldTable.isSetCatName() ? oldTable.getCatName() : getDefaultCatalog(conf)); @@ -789,6 +804,13 @@ void alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTa String tableName = normalizeIdentifier(oldTable.getTableName()); String newDbName = newTable.getDbName().toLowerCase(); String newTableName = normalizeIdentifier(newTable.getTableName()); + long txnId = -1; + String validWriteIds = null; + if (ec != null && ec.isSetProperties() && ec.getProperties().containsKey( + StatsSetupConst.VALID_WRITE_IDS)) { + txnId = Long.parseLong(ec.getProperties().get(StatsSetupConst.TXN_ID)); + validWriteIds = ec.getProperties().get(StatsSetupConst.VALID_WRITE_IDS); + } try { List oldCols = oldTable.getSd().getCols(); @@ -810,7 +832,7 @@ void alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTa oldColNames.add(oldCol.getName()); } - // Collect column stats which need to be rewritten and remove old stats + // Collect column stats which need to be rewritten and remove old stats. colStats = msdb.getTableColumnStatistics(catName, dbName, tableName, oldColNames); if (colStats == null) { updateColumnStats = false; @@ -845,7 +867,7 @@ void alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTa } // Change to new table and append stats for the new table - msdb.alterTable(catName, dbName, tableName, newTable); + msdb.alterTable(catName, dbName, tableName, newTable, txnId, validWriteIds); if (updateColumnStats && !newStatsObjs.isEmpty()) { ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); statsDesc.setDbName(newDbName); @@ -885,6 +907,7 @@ private ColumnStatistics updateOrGetPartitionColumnStats( oldColNames.add(oldCol.getName()); } List oldPartNames = Lists.newArrayList(oldPartName); + // Note: doesn't take txn stats into account. This method can only remove stats. List partsColStats = msdb.getPartitionColumnStatistics(catName, dbname, tblname, oldPartNames, oldColNames); assert (partsColStats.size() <= 1); diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 8ec5dfb4fa779fecdd24d6040e9770892c233341..5b3f711f092265abdb32f77cfa64764c821782ed 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -707,6 +707,10 @@ public static RawStore getMSForConf(Configuration conf) throws MetaException { @Override public TxnStore getTxnHandler() { + return getMsThreadTxnHandler(conf); + } + + public static TxnStore getMsThreadTxnHandler(Configuration conf) { TxnStore txn = threadLocalTxn.get(); if (txn == null) { txn = TxnUtils.getTxnStore(conf); @@ -4891,7 +4895,7 @@ public void alter_partitions(final String db_name, final String tbl_name, } @Override - public AlterPartitionsResponse alter_partitions_with_environment_context( + public AlterPartitionsResponse alter_partitions_with_environment_context_req( AlterPartitionsRequest req) throws TException { alter_partitions_with_environment_context( @@ -4902,6 +4906,16 @@ public AlterPartitionsResponse alter_partitions_with_environment_context( return new AlterPartitionsResponse(); } + // The old API we are keeping for backward compat. Not used within Hive. + @Deprecated + @Override + public void alter_partitions_with_environment_context(final String db_name, final String tbl_name, + final List new_parts, EnvironmentContext environmentContext) + throws TException { + alter_partitions_with_environment_context(db_name, tbl_name, new_parts, environmentContext, + -1, null, -1); + } + private void alter_partitions_with_environment_context(final String db_name, final String tbl_name, final List new_parts, EnvironmentContext environmentContext, long txnId, String writeIdList, long writeId) @@ -5654,9 +5668,12 @@ public TableStatsResult get_table_statistics_req(TableStatsRequest request) thro return result; } + @Deprecated @Override public ColumnStatistics get_partition_column_statistics(String dbName, String tableName, String partName, String colName) throws TException { + // Note: this method appears to be unused within Hive. + // It doesn't take txn stats into account. dbName = dbName.toLowerCase(); String[] parsedDbName = parseDbName(dbName, conf); tableName = tableName.toLowerCase(); @@ -7444,6 +7461,7 @@ public boolean set_aggr_stats_for(SetPartitionsStatsRequest request) throws TExc for (ColumnStatistics csOld : csOlds) { oldStatsMap.put(csOld.getStatsDesc().getPartName(), csOld); } + // another single call to get all the partition objects partitions = getMS().getPartitionsByNames(catName, dbName, tableName, partitionNames); for (int index = 0; index < partitionNames.size(); index++) { diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 30355e500de28edaee0b08ed80e2d0e4963d7723..8da6d2137115897433e78ae63e3b9d0129a1c29b 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -1921,7 +1921,7 @@ public void alter_partitions(String catName, String dbName, String tblName, req.setTxnId(txnId); req.setValidWriteIdList(writeIdList); req.setWriteId(writeId); - client.alter_partitions_with_environment_context(req); + client.alter_partitions_with_environment_context_req(req); } @Override diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 750040a9145242aca626a1b28f1b043ca8e7b262..39b56870b62eeb1b14cf77ce20d8da575a321d1c 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -1199,8 +1199,7 @@ public void createTable(Table tbl) throws InvalidObjectException, MetaException mtbl = convertToMTable(tbl); if (TxnUtils.isTransactionalTable(tbl)) { - mtbl.setTxnId(tbl.getTxnId()); - mtbl.setWriteIdList(tbl.getValidWriteIdList()); + mtbl.setWriteId(tbl.getWriteId()); } pm.makePersistent(mtbl); @@ -1312,6 +1311,7 @@ public boolean dropTable(String catName, String dbName, String tableName) TableName.getQualified(catName, dbName, tableName)); } + // TODO## remove? unused Table table = convertToTable(tbl); List tabConstraints = listAllTableConstraintsWithOptionalConstraintName( @@ -1448,10 +1448,10 @@ public Table getTable(String catName, String dbName, String tableName, if (tbl != null && TxnUtils.isTransactionalTable(tbl) && tbl.getPartitionKeysSize() == 0) { - if (isCurrentStatsValidForTheQuery(mtable, txnId, writeIdList, -1, false)) { - tbl.setIsStatsCompliant(IsolationLevelCompliance.YES); + if (isCurrentStatsValidForTheQuery(mtable, txnId, writeIdList)) { + tbl.setIsStatsCompliant(true); } else { - tbl.setIsStatsCompliant(IsolationLevelCompliance.NO); + tbl.setIsStatsCompliant(false); // Do not make persistent the following state since it is the query specific (not global). StatsSetupConst.setBasicStatsState(tbl.getParameters(), StatsSetupConst.FALSE); LOG.info("Removed COLUMN_STATS_ACCURATE from Table's parameters."); @@ -1987,8 +1987,6 @@ private MTable convertToMTable(Table tbl) throws InvalidObjectException, tbl.getViewOriginalText(), tbl.getViewExpandedText(), tbl.isRewriteEnabled(), tableType); if (TxnUtils.isTransactionalTable(tbl)) { - mtable.setTxnId(tbl.getTxnId()); - mtable.setWriteIdList(tbl.getValidWriteIdList()); mtable.setWriteId(tbl.getWriteId()); } return mtable; @@ -2456,10 +2454,10 @@ public Partition getPartition(String catName, String dbName, String tableName, // statistics in the metastore comply with the client query's snapshot isolation. if (writeIdList != null) { if (TxnUtils.isTransactionalTable(table.getParameters())) { - if (isCurrentStatsValidForTheQuery(mpart, txnId, writeIdList, -1, false)) { - part.setIsStatsCompliant(IsolationLevelCompliance.YES); + if (isCurrentStatsValidForTheQuery(mpart, txnId, writeIdList)) { + part.setIsStatsCompliant(true); } else { - part.setIsStatsCompliant(IsolationLevelCompliance.NO); + part.setIsStatsCompliant(false); // Do not make persistent the following state since it is query specific (not global). StatsSetupConst.setBasicStatsState(part.getParameters(), StatsSetupConst.FALSE); LOG.info("Removed COLUMN_STATS_ACCURATE from Partition object's parameters."); @@ -2600,8 +2598,6 @@ private MPartition convertToMPart(Partition part, MTable mt, boolean useTableCD) .getCreateTime(), part.getLastAccessTime(), msd, part.getParameters()); if (TxnUtils.isTransactionalTable(mt.getParameters())) { - mpart.setTxnId(part.getTxnId()); - mpart.setWriteIdList(part.getValidWriteIdList()); mpart.setWriteId(part.getWriteId()); } return mpart; @@ -4095,8 +4091,8 @@ private String makeParameterDeclarationStringObj(Map params) { } @Override - public void alterTable(String catName, String dbname, String name, Table newTable) - throws InvalidObjectException, MetaException { + public void alterTable(String catName, String dbname, String name, Table newTable, + long queryTxnId, String queryValidWriteIds) throws InvalidObjectException, MetaException { boolean success = false; boolean registerCreationSignature = false; try { @@ -4139,18 +4135,14 @@ public void alterTable(String catName, String dbname, String name, Table newTabl // If transactional, update MTable to have txnId and the writeIdList // for the current Stats updater query. - if (newTable.getValidWriteIdList() != null && - TxnUtils.isTransactionalTable(newTable)) { + if (TxnUtils.isTransactionalTable(newTable) && queryValidWriteIds != null) { // Check concurrent INSERT case and set false to the flag. - if (!isCurrentStatsValidForTheQuery(oldt, newt.getTxnId(), newt.getWriteIdList(), - newt.getWriteId(), true)) { + if (!isCurrentStatsValidForTheQuery(oldt, queryTxnId, queryValidWriteIds)) { StatsSetupConst.setBasicStatsState(oldt.getParameters(), StatsSetupConst.FALSE); LOG.info("Removed COLUMN_STATS_ACCURATE from the parameters of the table " + dbname + "." + name + ". will be made persistent."); } - oldt.setTxnId(newTable.getTxnId()); oldt.setWriteId(newTable.getWriteId()); - oldt.setWriteIdList(newTable.getValidWriteIdList()); } // commit the changes @@ -4200,7 +4192,7 @@ public void updateCreationMetadata(String catName, String dbname, String tablena * @throws MetaException */ private MColumnDescriptor alterPartitionNoTxn(String catName, String dbname, String name, - List part_vals, Partition newPart) + List part_vals, Partition newPart, long queryTxnId, String queryValidWriteIds) throws InvalidObjectException, MetaException { catName = normalizeIdentifier(catName); name = normalizeIdentifier(name); @@ -4228,19 +4220,16 @@ private MColumnDescriptor alterPartitionNoTxn(String catName, String dbname, Str if (newp.getLastAccessTime() != oldp.getLastAccessTime()) { oldp.setLastAccessTime(newp.getLastAccessTime()); } + // If transactional, add/update the MUPdaterTransaction // for the current updater query. - if (newPart.getValidWriteIdList() != null && - TxnUtils.isTransactionalTable(table.getParameters())) { + if (queryValidWriteIds != null && TxnUtils.isTransactionalTable(table.getParameters())) { // Check concurrent INSERT case and set false to the flag. - if (!isCurrentStatsValidForTheQuery(oldp, newp.getTxnId(), newp.getWriteIdList(), - newp.getWriteId(), true)) { + if (!isCurrentStatsValidForTheQuery(oldp, queryTxnId, queryValidWriteIds)) { StatsSetupConst.setBasicStatsState(oldp.getParameters(), StatsSetupConst.FALSE); LOG.info("Removed COLUMN_STATS_ACCURATE from the parameters of the partition " + dbname + "." + name + "." + oldp.getPartitionName() + " will be made persistent."); } - oldp.setTxnId(newPart.getTxnId()); - oldp.setWriteIdList(newPart.getValidWriteIdList()); oldp.setWriteId(newPart.getWriteId()); } return oldCD; @@ -4248,12 +4237,16 @@ private MColumnDescriptor alterPartitionNoTxn(String catName, String dbname, Str @Override public void alterPartition(String catName, String dbname, String name, List part_vals, - Partition newPart) throws InvalidObjectException, MetaException { + Partition newPart, long queryTxnId, String queryValidWriteIds) throws InvalidObjectException, MetaException { boolean success = false; Exception e = null; try { openTransaction(); - MColumnDescriptor oldCd = alterPartitionNoTxn(catName, dbname, name, part_vals, newPart); + if (newPart.isSetWriteId()) { + LOG.warn("Alter partitions with write ID called without transaction information"); + } + MColumnDescriptor oldCd = alterPartitionNoTxn( + catName, dbname, name, part_vals, newPart, queryTxnId, queryValidWriteIds); removeUnusedColumnDescriptor(oldCd); // commit the changes success = commitTransaction(); @@ -4275,8 +4268,8 @@ public void alterPartition(String catName, String dbname, String name, List> part_vals, List newParts, - long txnId, String writeIdList, long writeId) - throws InvalidObjectException, MetaException { + long writeId, long queryTxnId, String queryWriteIdList) + throws InvalidObjectException, MetaException { boolean success = false; Exception e = null; try { @@ -4285,12 +4278,12 @@ public void alterPartitions(String catName, String dbname, String name, Set oldCds = new HashSet<>(); for (Partition tmpPart: newParts) { List tmpPartVals = part_val_itr.next(); - if (txnId > 0) { - tmpPart.setTxnId(txnId); - tmpPart.setValidWriteIdList(writeIdList); + // We don't reset write ID when we invalidate stats; we unset the json boolean. + if (writeId > 0) { tmpPart.setWriteId(writeId); } - MColumnDescriptor oldCd = alterPartitionNoTxn(catName, dbname, name, tmpPartVals, tmpPart); + MColumnDescriptor oldCd = alterPartitionNoTxn( + catName, dbname, name, tmpPartVals, tmpPart, queryTxnId, queryWriteIdList); if (oldCd != null) { oldCds.add(oldCd); } @@ -8479,10 +8472,7 @@ public boolean updatePartitionColumnStatistics(ColumnStatistics colStats, List getMTableColumnStatistics( - Table table, - List colNames, - QueryWrapper queryWrapper) + private List getMTableColumnStatistics(Table table, List colNames, QueryWrapper queryWrapper) throws MetaException { if (colNames == null || colNames.isEmpty()) { return Collections.emptyList(); @@ -8562,6 +8552,7 @@ public ColumnStatistics getTableColumnStatistics( String dbName, String tableName, List colNames) throws MetaException, NoSuchObjectException { + // Note: this will get stats without verifying ACID. return getTableColumnStatisticsInternal( catName, dbName, tableName, colNames, true, true); } @@ -8574,20 +8565,16 @@ public ColumnStatistics getTableColumnStatistics( List colNames, long txnId, String writeIdList) throws MetaException, NoSuchObjectException { - IsolationLevelCompliance iLL = IsolationLevelCompliance.UNKNOWN; + Boolean iLL = null; // If the current stats in the metastore doesn't comply with // the isolation level of the query, set No to the compliance flag. if (writeIdList != null) { MTable table = this.getMTable(catName, dbName, tableName); - if (!isCurrentStatsValidForTheQuery(table, txnId, writeIdList, -1, false)) { - iLL = IsolationLevelCompliance.NO; - } else { - iLL = IsolationLevelCompliance.YES; - } + iLL = isCurrentStatsValidForTheQuery(table, txnId, writeIdList); } ColumnStatistics cS = getTableColumnStatisticsInternal( catName, dbName, tableName, colNames, true, true); - if (cS != null) { + if (cS != null && iLL != null) { cS.setIsStatsCompliant(iLL); } return cS; @@ -8636,8 +8623,7 @@ protected ColumnStatistics getJdoResult( @Override public List getPartitionColumnStatistics(String catName, String dbName, String tableName, List partNames, List colNames) throws MetaException, NoSuchObjectException { - // TODO: this will get stats without verifying ACID. Not clear when this can be valid... - // We need to check that this is not called on a txn table. + // Note: this will get stats without verifying ACID. return getPartitionColumnStatisticsInternal( catName, dbName, tableName, partNames, colNames, true, true); } @@ -8656,15 +8642,17 @@ protected ColumnStatistics getJdoResult( LOG.warn("The given partNames does not have any name."); return null; } + // TODO## this is not correct; stats updater patch will fix it to return stats for valid partitions, + // and no stats for invalid. Remove this comment when merging that patch. // Loop through the given "partNames" list // checking isolation-level-compliance of each partition column stats. for(String partName : partNames) { MPartition mpart = getMPartition(catName, dbName, tableName, Warehouse.getPartValuesFromPartName(partName)); - if (!isCurrentStatsValidForTheQuery(mpart, txnId, writeIdList, -1, false)) { - LOG.debug("The current metastore transactional partition column statistics " + - "for " + dbName + "." + tableName + "." + mpart.getPartitionName() + " is not valid " + - "for the current query."); - return null; + if (!isCurrentStatsValidForTheQuery(mpart, txnId, writeIdList)) { + LOG.debug("The current metastore transactional partition column statistics for {}.{}.{} " + + "(write ID {}) are not valid for current query ({} {})", dbName, tableName, + mpart.getPartitionName(), mpart.getWriteId(), txnId, writeIdList); + return Lists.newArrayList(); } } } @@ -8731,11 +8719,14 @@ public AggrStats get_aggr_stats_for(String catName, String dbName, String tblNam LOG.warn("The given partNames does not have any name."); return null; } + + // TODO: this should probably also return stats for partitions with valid stats, + // and no stats for partitions with invalid stats. // Loop through the given "partNames" list // checking isolation-level-compliance of each partition column stats. for(String partName : partNames) { MPartition mpart = getMPartition(catName, dbName, tblName, Warehouse.getPartValuesFromPartName(partName)); - if (!isCurrentStatsValidForTheQuery(mpart, txnId, writeIdList, -1, false)) { + if (!isCurrentStatsValidForTheQuery(mpart, txnId, writeIdList)) { LOG.debug("The current metastore transactional partition column statistics " + "for " + dbName + "." + tblName + "." + mpart.getPartitionName() + " is not valid " + "for the current query."); @@ -12256,12 +12247,9 @@ public int deleteRuntimeStats(int maxRetainSecs) throws MetaException { * @Precondition "tbl" should be retrieved from the TBLS table. */ private boolean isCurrentStatsValidForTheQuery( - MTable tbl, long queryTxnId, String queryValidWriteIdList, - long queryWriteId, boolean checkConcurrentWrites) - throws MetaException { - return isCurrentStatsValidForTheQuery(tbl.getParameters(), tbl.getTxnId(), - tbl.getWriteIdList(),tbl.getWriteId(), - queryTxnId, queryValidWriteIdList, queryWriteId, checkConcurrentWrites); + MTable tbl, long queryTxnId, String queryValidWriteIdList) throws MetaException { + return isCurrentStatsValidForTheQuery(tbl.getDatabase().getName(), tbl.getTableName(), + tbl.getParameters(), tbl.getWriteId(), queryTxnId, queryValidWriteIdList); } /** @@ -12279,58 +12267,25 @@ private boolean isCurrentStatsValidForTheQuery( * @Precondition "part" should be retrieved from the PARTITIONS table. */ private boolean isCurrentStatsValidForTheQuery( - MPartition part, long queryTxnId, String queryValidWriteIdList, - long queryWriteId, boolean checkConcurrentWrites) + MPartition part, long queryTxnId, String queryValidWriteIdList) throws MetaException { - return isCurrentStatsValidForTheQuery(part.getParameters(), part.getTxnId(), - part.getWriteIdList(), part.getWriteId(), - queryTxnId, queryValidWriteIdList, queryWriteId, checkConcurrentWrites); + return isCurrentStatsValidForTheQuery(part.getTable().getDatabase().getName(), + part.getTable().getTableName(), part.getParameters(), part.getWriteId(), + queryTxnId, queryValidWriteIdList); } - private boolean isCurrentStatsValidForTheQuery( - Map statsParams, long statsTxnId, - String statsWriteIdList, long statsWriteId, - long queryTxnId, String queryValidWriteIdList, - long queryWriteId, boolean checkConcurrentWrites) - throws MetaException { - // If checkConcurrentWrites is true and - // statsWriteId or queryWriteId is -1 or 0, - // return true since -1 or 0 is not a valid writeId. - if (checkConcurrentWrites) { - if (queryWriteId < 1) { - LOG.error("Cannot check for concurrent inserts without a valid query write ID"); - return false; - } - if (statsWriteId < 1) { - return true; // TODO: this is questionable, too - } - } - + private boolean isCurrentStatsValidForTheQuery(String dbName, String tblName, + Map statsParams, long statsWriteId, long queryTxnId, + String queryValidWriteIdList) throws MetaException { // Note: can be changed to debug/info to verify the calls. - LOG.trace("Called with stats {}, {}; query {}, {}; checkConcurrentWrites {}", - statsTxnId, statsWriteIdList, queryTxnId, queryValidWriteIdList, checkConcurrentWrites); + LOG.trace("Called with stats write ID {}; query {}, {}; params {}", + statsWriteId, queryTxnId, queryValidWriteIdList, statsParams); // if statsWriteIdList is null, // return true since the stats does not seem to be transactional. - if (statsWriteIdList == null) { + if (statsWriteId < 1) { return true; } - // If the current query is a stats updater, then we can return true - // to avoid implementing a logic inside TxnIdUtils.checkEquivalentWriteIds(). - if (statsTxnId == queryTxnId) { - return true; - } - - // If the Metastore stats's writer transaction is open or aborted - // we should return false. - try { - if (TxnDbUtil.isOpenOrAbortedTransaction(conf, statsTxnId)) { - return false; - } - } catch (Exception e) { - throw new MetaException("Cannot check transaction state."); - } - // This COLUMN_STATS_ACCURATE(CSA) state checking also includes the case that the stats is // written by an aborted transaction but TXNS has no entry for the transaction // after compaction. @@ -12338,16 +12293,26 @@ private boolean isCurrentStatsValidForTheQuery( return false; } + // TODO## NUM_FILES could also be set to 0 by invalid update. We need to have a negative test. Or remove this and fix stuff. // If the NUM_FILES of the table/partition is 0, return 'true' from this method. // Since newly initialized empty table has 0 for the parameter. if (Long.parseLong(statsParams.get(StatsSetupConst.NUM_FILES)) == 0) { return true; } - ValidWriteIdList list4Stats = new ValidReaderWriteIdList(statsWriteIdList); ValidWriteIdList list4TheQuery = new ValidReaderWriteIdList(queryValidWriteIdList); + // Just check if the write ID is valid. If it's valid (i.e. we are allowed to see it), + // that means it cannot possibly be a concurrent write. If it's not valid (we are not + // allowed to see it), that means it's either concurrent or aborted, same thing for us. + if (list4TheQuery.isWriteIdValid(statsWriteId)) { + return true; + } - return !checkConcurrentWrites ? TxnIdUtils.checkEquivalentWriteIds(list4Stats, list4TheQuery) : - !TxnIdUtils.areTheseConcurrentWrites(list4Stats, statsWriteId, list4TheQuery, queryWriteId); + // This assumes that all writes within the same txn are sequential and can see each other. + // TODO## Not clear if we need this check; each next write should have the previous + // one in its writeIdList; verify w/Eugene. + long statsTxnId = HiveMetaStore.HMSHandler.getMsThreadTxnHandler(conf).getTxnIdForWriteId( + dbName, tblName, statsWriteId); + return (statsTxnId == queryTxnId); } } diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index 4800c8af2c69cc498da25cb89298cdd172b1a767..ccd1311def34370034d7c2e76e48f9d711d5a40d 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -361,7 +361,8 @@ boolean dropPartition(String catName, String dbName, String tableName, * @throws InvalidObjectException The new table object is invalid. * @throws MetaException something went wrong, usually in the RDBMS or storage. */ - void alterTable(String catName, String dbname, String name, Table newTable) + void alterTable(String catName, String dbname, String name, Table newTable, + long queryTxnId, String queryValidWriteIds) throws InvalidObjectException, MetaException; /** @@ -502,7 +503,8 @@ PartitionValuesResponse listPartitionValues(String catName, String db_name, Stri * @throws MetaException error accessing the RDBMS. */ void alterPartition(String catName, String db_name, String tbl_name, List part_vals, - Partition new_part) throws InvalidObjectException, MetaException; + Partition new_part, long queryTxnId, String queryValidWriteIds) + throws InvalidObjectException, MetaException; /** * Alter a set of partitions. @@ -521,8 +523,8 @@ void alterPartition(String catName, String db_name, String tbl_name, List> part_vals_list, List new_parts, - long txnId, String writeIdList, long writeId) + List> part_vals_list, List new_parts, long writeId, + long queryTxnId, String queryValidWriteIds) throws InvalidObjectException, MetaException; /** diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index 42cf485cfda4c48a3c34537528ea21f62f5ab536..f626b37b211df5cbdf816d817ebb49fd74390903 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -279,6 +279,7 @@ static void prewarm(RawStore rawStore) { rawStore.getTableColumnStatistics(catName, dbName, tblName, colNames); Deadline.stopTimer(); } + // TODO## should this take write ID into account? or at least cache write ID to verify? // If the table could not cached due to memory limit, stop prewarm boolean isSuccess = sharedCache.populateTableInCache(table, tableColStats, partitions, partitionColStats, aggrStatsAllPartitions, aggrStatsAllButDefaultPartition); @@ -550,6 +551,7 @@ private void updateTableColStats(RawStore rawStore, String catName, String dbNam rawStore.getTableColumnStatistics(catName, dbName, tblName, colNames); Deadline.stopTimer(); if (tableColStats != null) { + // TODO## should this take write ID into account? or at least cache write ID to verify? sharedCache.refreshTableColStatsInCache(StringUtils.normalizeIdentifier(catName), StringUtils.normalizeIdentifier(dbName), StringUtils.normalizeIdentifier(tblName), tableColStats.getStatsObj()); @@ -580,6 +582,7 @@ private void updateTablePartitionColStats(RawStore rawStore, String catName, Str List partNames = rawStore.listPartitionNames(catName, dbName, tblName, (short) -1); // Get partition column stats for this table Deadline.startTimer("getPartitionColumnStatistics"); + // TODO## should this take write ID into account? or at least cache write ID to verify? List partitionColStats = rawStore.getPartitionColumnStatistics(catName, dbName, tblName, partNames, colNames); Deadline.stopTimer(); @@ -1006,9 +1009,9 @@ public void dropPartitions(String catName, String dbName, String tblName, List partVals, - Partition newPart) throws InvalidObjectException, MetaException { - rawStore.alterPartition(catName, dbName, tblName, partVals, newPart); + Partition newPart, long queryTxnId, String queryValidWriteIds) + throws InvalidObjectException, MetaException { + rawStore.alterPartition(catName, dbName, tblName, partVals, newPart, queryTxnId, queryValidWriteIds); catName = normalizeIdentifier(catName); dbName = normalizeIdentifier(dbName); tblName = normalizeIdentifier(tblName); @@ -1172,9 +1176,10 @@ public void alterPartition(String catName, String dbName, String tblName, List> partValsList, List newParts, - long txnId, String writeIdList, long writeId) + long writeId, long txnId, String validWriteIds) throws InvalidObjectException, MetaException { - rawStore.alterPartitions(catName, dbName, tblName, partValsList, newParts, txnId, writeIdList, writeId); + rawStore.alterPartitions( + catName, dbName, tblName, partValsList, newParts, writeId, txnId, validWriteIds); catName = normalizeIdentifier(catName); dbName = normalizeIdentifier(dbName); tblName = normalizeIdentifier(tblName); diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MPartition.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MPartition.java index b5218f36a5b8667539657443e663d894865b3093..267c9e8e5acd7f3b3666f9a68780091c5e792380 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MPartition.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MPartition.java @@ -30,9 +30,7 @@ private int lastAccessTime; private MStorageDescriptor sd; private Map parameters; - private long txnId; private long writeId; - private String writeIdList; public MPartition() {} @@ -154,14 +152,6 @@ public void setCreateTime(int createTime) { this.createTime = createTime; } - public long getTxnId() { - return txnId; - } - - public void setTxnId(long txnId) { - this.txnId = txnId; - } - public long getWriteId() { return writeId; } @@ -169,12 +159,4 @@ public long getWriteId() { public void setWriteId(long writeId) { this.writeId = writeId; } - - public String getWriteIdList() { - return writeIdList; - } - - public void setWriteIdList(String writeIdList) { - this.writeIdList = writeIdList; - } } diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java index 269325175d186b39d2e14519cb59866500ef9230..deeb97133d4aeb362c892e4a08346189eec26b09 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java @@ -38,9 +38,7 @@ private String viewExpandedText; private boolean rewriteEnabled; private String tableType; - private long txnId; private long writeId; - private String writeIdList; public MTable() {} @@ -275,14 +273,6 @@ public String getTableType() { return tableType; } - public long getTxnId() { - return txnId; - } - - public void setTxnId(long txnId) { - this.txnId = txnId; - } - public long getWriteId() { return writeId; } @@ -290,12 +280,4 @@ public long getWriteId() { public void setWriteId(long writeId) { this.writeId = writeId; } - - public String getWriteIdList() { - return writeIdList; - } - - public void setWriteIdList(String writeIdList) { - this.writeIdList = writeIdList; - } } diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java index 7f1c89b0c9736a4bfbbe4f081dcfe32408357725..938e5b41e9064b3c78ff82a3a2e89c2b8227ae13 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java @@ -203,8 +203,8 @@ public static void prepDb(Configuration conf) throws Exception { " \"OWNER\" VARCHAR(767), \"OWNER_TYPE\" VARCHAR(10), \"RETENTION\" INTEGER NOT NULL, " + " \"SD_ID\" BIGINT, \"TBL_NAME\" VARCHAR(256), \"TBL_TYPE\" VARCHAR(128), " + " \"VIEW_EXPANDED_TEXT\" LONG VARCHAR, \"VIEW_ORIGINAL_TEXT\" LONG VARCHAR, " + - " \"IS_REWRITE_ENABLED\" CHAR(1) NOT NULL DEFAULT \'N\', \"TXN_ID\" BIGINT DEFAULT 0, " + - " \"WRITE_ID\" BIGINT DEFAULT 0, \"WRITEID_LIST\" CLOB, " + + " \"IS_REWRITE_ENABLED\" CHAR(1) NOT NULL DEFAULT \'N\', " + + " \"WRITE_ID\" BIGINT DEFAULT 0, " + " PRIMARY KEY (TBL_ID))" ); } catch (SQLException e) { @@ -219,8 +219,8 @@ public static void prepDb(Configuration conf) throws Exception { stmt.execute("CREATE TABLE \"APP\".\"PARTITIONS\" (" + " \"PART_ID\" BIGINT NOT NULL, \"CREATE_TIME\" INTEGER NOT NULL, " + " \"LAST_ACCESS_TIME\" INTEGER NOT NULL, \"PART_NAME\" VARCHAR(767), " + - " \"SD_ID\" BIGINT, \"TBL_ID\" BIGINT, \"TXN_ID\" BIGINT DEFAULT 0, " + - " \"WRITE_ID\" BIGINT DEFAULT 0, \"WRITEID_LIST\" CLOB, " + + " \"SD_ID\" BIGINT, \"TBL_ID\" BIGINT, " + + " \"WRITE_ID\" BIGINT DEFAULT 0, " + " PRIMARY KEY (PART_ID))" ); } catch (SQLException e) { diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index 361ede54efcd739516285e7b6a85e1122e8b870a..f00e5382c0c59b3aecc262af1de72901c94d14d2 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -86,6 +86,7 @@ import org.apache.hadoop.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import com.google.common.annotations.VisibleForTesting; /** @@ -421,6 +422,7 @@ public GetOpenTxnsInfoResponse getOpenTxnsInfo() throws MetaException { return getOpenTxnsInfo(); } } + @Override @RetrySemantics.ReadOnly public GetOpenTxnsResponse getOpenTxns() throws MetaException { @@ -2300,6 +2302,47 @@ long generateCompactionQueueId(Statement stmt) throws SQLException, MetaExceptio return id; } } + + @Override + @RetrySemantics.ReadOnly + public long getTxnIdForWriteId( + String dbName, String tblName, long writeId) throws MetaException { + try { + Connection dbConn = null; + Statement stmt = null; + try { + /** + * This runs at READ_COMMITTED for exactly the same reason as {@link #getOpenTxnsInfo()} + */ + dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); + stmt = dbConn.createStatement(); + + String query = "select t2w_txnid from TXN_TO_WRITE_ID where" + + " t2w_database = " + quoteString(dbName) + + " and t2w_table = " + quoteString(tblName) + + " and t2w_writeid = " + writeId; + LOG.debug("Going to execute query <" + query + ">"); + ResultSet rs = stmt.executeQuery(query); + long txnId = -1; + if (rs.next()) { + txnId = rs.getLong(1); + } + dbConn.rollback(); + return txnId; + } catch (SQLException e) { + LOG.debug("Going to rollback"); + rollbackDBConn(dbConn); + checkRetryable(dbConn, e, "getTxnIdForWriteId"); + throw new MetaException("Unable to select from transaction database, " + + StringUtils.stringifyException(e)); + } finally { + close(null, stmt, dbConn); + } + } catch (RetryException e) { + return getTxnIdForWriteId(dbName, tblName, writeId); + } + } + @Override @RetrySemantics.Idempotent public CompactionResponse compact(CompactionRequest rqst) throws MetaException { diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java index ef447e1c99802f1bd198465a2aa1ac9f53ac18db..1df45d00e50a39db123b63f237031622c511d471 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.metastore.txn; import com.google.common.annotations.VisibleForTesting; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configurable; @@ -133,6 +134,11 @@ void commitTxn(CommitTxnRequest rqst) BasicTxnInfo getFirstCompletedTransactionForTableAfterCommit( String inputDbName, String inputTableName, ValidWriteIdList txnList) throws MetaException; + + @RetrySemantics.ReadOnly + long getTxnIdForWriteId(String dbName, String tblName, long writeId) + throws MetaException; + /** * Gets the list of valid write ids for the given table wrt to current txn * @param rqst info on transaction and list of table names associated with given transaction diff --git standalone-metastore/src/main/resources/package.jdo standalone-metastore/src/main/resources/package.jdo index 4746aee4c5d71af82b34d05623860c9b43875592..3ea4a7a723a08670ab50d83f58ee37aa5833b729 100644 --- standalone-metastore/src/main/resources/package.jdo +++ standalone-metastore/src/main/resources/package.jdo @@ -210,15 +210,9 @@ - - - - - - @@ -498,15 +492,9 @@ - - - - - - diff --git standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql index 1af5dbccf54eb16c6a277f5f6dbb8eb37419ad99..714559df82086058ae40bf8666d08ee3aee0821a 100644 --- standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql +++ standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql @@ -47,7 +47,7 @@ CREATE TABLE "APP"."IDXS" ("INDEX_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT CREATE TABLE "APP"."INDEX_PARAMS" ("INDEX_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" VARCHAR(4000)); -CREATE TABLE "APP"."PARTITIONS" ("PART_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "LAST_ACCESS_TIME" INTEGER NOT NULL, "PART_NAME" VARCHAR(767), "SD_ID" BIGINT, "TBL_ID" BIGINT, "TXN_ID" BIGINT DEFAULT 0, "WRITE_ID" BIGINT DEFAULT 0, "WRITEID_LIST" CLOB); +CREATE TABLE "APP"."PARTITIONS" ("PART_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "LAST_ACCESS_TIME" INTEGER NOT NULL, "PART_NAME" VARCHAR(767), "SD_ID" BIGINT, "TBL_ID" BIGINT, "WRITE_ID" BIGINT DEFAULT 0); CREATE TABLE "APP"."SERDES" ("SERDE_ID" BIGINT NOT NULL, "NAME" VARCHAR(128), "SLIB" VARCHAR(4000), "DESCRIPTION" VARCHAR(4000), "SERIALIZER_CLASS" VARCHAR(4000), "DESERIALIZER_CLASS" VARCHAR(4000), SERDE_TYPE INTEGER); @@ -75,7 +75,7 @@ CREATE TABLE "APP"."COLUMNS" ("SD_ID" BIGINT NOT NULL, "COMMENT" VARCHAR(256), " CREATE TABLE "APP"."ROLES" ("ROLE_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "OWNER_NAME" VARCHAR(128), "ROLE_NAME" VARCHAR(128)); -CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER" VARCHAR(767), "OWNER_TYPE" VARCHAR(10), "RETENTION" INTEGER NOT NULL, "SD_ID" BIGINT, "TBL_NAME" VARCHAR(256), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT" LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR, "IS_REWRITE_ENABLED" CHAR(1) NOT NULL DEFAULT 'N', "TXN_ID" BIGINT DEFAULT 0, "WRITE_ID" BIGINT DEFAULT 0, "WRITEID_LIST" CLOB); +CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER" VARCHAR(767), "OWNER_TYPE" VARCHAR(10), "RETENTION" INTEGER NOT NULL, "SD_ID" BIGINT, "TBL_NAME" VARCHAR(256), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT" LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR, "IS_REWRITE_ENABLED" CHAR(1) NOT NULL DEFAULT 'N', "WRITE_ID" BIGINT DEFAULT 0); CREATE TABLE "APP"."PARTITION_KEYS" ("TBL_ID" BIGINT NOT NULL, "PKEY_COMMENT" VARCHAR(4000), "PKEY_NAME" VARCHAR(128) NOT NULL, "PKEY_TYPE" VARCHAR(767) NOT NULL, "INTEGER_IDX" INTEGER NOT NULL); diff --git standalone-metastore/src/main/sql/derby/upgrade-3.1.0-to-4.0.0.derby.sql standalone-metastore/src/main/sql/derby/upgrade-3.1.0-to-4.0.0.derby.sql index 41ffe1ce69bbe98b30c156cdc0350265212445c0..d4fb2990f202b4af1c963d4207defc4c5dc6f12c 100644 --- standalone-metastore/src/main/sql/derby/upgrade-3.1.0-to-4.0.0.derby.sql +++ standalone-metastore/src/main/sql/derby/upgrade-3.1.0-to-4.0.0.derby.sql @@ -1,10 +1,6 @@ -- Upgrade MetaStore schema from 3.1.0 to 4.0.0 -- HIVE-19416 -ALTER TABLE "APP"."TBLS" ADD WRITEID_LIST CLOB; -ALTER TABLE "APP"."TBLS" ADD TXN_ID bigint DEFAULT 0; ALTER TABLE "APP"."TBLS" ADD WRITE_ID bigint DEFAULT 0; -ALTER TABLE "APP"."PARTITIONS" ADD WRITEID_LIST CLOB; -ALTER TABLE "APP"."PARTITIONS" ADD TXN_ID bigint DEFAULT 0; ALTER TABLE "APP"."PARTITIONS" ADD WRITE_ID bigint DEFAULT 0; -- This needs to be the last thing done. Insert any changes above this line. diff --git standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql index 6ce2639b9b58ce423d3497ef9fcb187b9618502d..4c3f131c3655f03cd09057dbfd90cf4488a896b5 100644 --- standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql +++ standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql @@ -146,9 +146,7 @@ CREATE TABLE PARTITIONS PART_NAME nvarchar(767) NULL, SD_ID bigint NULL, TBL_ID bigint NULL, - TXN_ID bigint NULL, - WRITEID_LIST text NULL -); + WRITE_ID bigint NULL); ALTER TABLE PARTITIONS ADD CONSTRAINT PARTITIONS_PK PRIMARY KEY (PART_ID); @@ -380,10 +378,7 @@ CREATE TABLE TBLS VIEW_EXPANDED_TEXT text NULL, VIEW_ORIGINAL_TEXT text NULL, IS_REWRITE_ENABLED bit NOT NULL DEFAULT 0, - TXN_ID bigint NULL, - WRITE_ID bigint NULL, - WRITEID_LIST text NULL -); + WRITE_ID bigint NULL); ALTER TABLE TBLS ADD CONSTRAINT TBLS_PK PRIMARY KEY (TBL_ID); diff --git standalone-metastore/src/main/sql/mssql/upgrade-3.1.0-to-4.0.0.mssql.sql standalone-metastore/src/main/sql/mssql/upgrade-3.1.0-to-4.0.0.mssql.sql index 44939d2ce364c3c6030aa6f3968762874f50aea4..acc9361246da04e594b663f56eccdc992c9ce670 100644 --- standalone-metastore/src/main/sql/mssql/upgrade-3.1.0-to-4.0.0.mssql.sql +++ standalone-metastore/src/main/sql/mssql/upgrade-3.1.0-to-4.0.0.mssql.sql @@ -1,11 +1,7 @@ SELECT 'Upgrading MetaStore schema from 3.1.0 to 4.0.0' AS MESSAGE; -- HIVE-19416 -ALTER TABLE TBLS ADD WRITEID_LIST text NULL; -ALTER TABLE TBLS ADD TXN_ID bigint NULL; ALTER TABLE TBLS ADD WRITE_ID bigint NULL; -ALTER TABLE PARTITIONS ADD WRITEID_LIST text NULL; -ALTER TABLE PARTITIONS ADD TXN_ID bigint NULL; ALTER TABLE PARTITIONS ADD WRITE_ID bigint NULL; -- These lines need to be last. Insert any changes above. diff --git standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql index 86fef268be6337e422f0be3c5675d6a4cdcb272d..be8dd9119a6e45acf123452ff9f24e41c2146203 100644 --- standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql +++ standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql @@ -224,9 +224,7 @@ CREATE TABLE IF NOT EXISTS `PARTITIONS` ( `PART_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, `SD_ID` bigint(20) DEFAULT NULL, `TBL_ID` bigint(20) DEFAULT NULL, - `TXN_ID` bigint(20) DEFAULT 0, `WRITE_ID` bigint(20) DEFAULT 0, - `WRITEID_LIST` text DEFAULT NULL, PRIMARY KEY (`PART_ID`), UNIQUE KEY `UNIQUEPARTITION` (`PART_NAME`,`TBL_ID`), KEY `PARTITIONS_N49` (`TBL_ID`), @@ -632,9 +630,7 @@ CREATE TABLE IF NOT EXISTS `TBLS` ( `VIEW_EXPANDED_TEXT` mediumtext, `VIEW_ORIGINAL_TEXT` mediumtext, `IS_REWRITE_ENABLED` bit(1) NOT NULL DEFAULT 0, - `TXN_ID` bigint(20) DEFAULT 0, `WRITE_ID` bigint(20) DEFAULT 0, - `WRITEID_LIST` text DEFAULT NULL, PRIMARY KEY (`TBL_ID`), UNIQUE KEY `UNIQUETABLE` (`TBL_NAME`,`DB_ID`), KEY `TBLS_N50` (`SD_ID`), diff --git standalone-metastore/src/main/sql/mysql/upgrade-3.1.0-to-4.0.0.mysql.sql standalone-metastore/src/main/sql/mysql/upgrade-3.1.0-to-4.0.0.mysql.sql index 9228cca3db6f874e6e5f96f873039659076f7716..89265ad286b508e4c2e14339e8b7d8ecd2e1ed15 100644 --- standalone-metastore/src/main/sql/mysql/upgrade-3.1.0-to-4.0.0.mysql.sql +++ standalone-metastore/src/main/sql/mysql/upgrade-3.1.0-to-4.0.0.mysql.sql @@ -1,12 +1,8 @@ SELECT 'Upgrading MetaStore schema from 3.1.0 to 4.0.0' AS ' '; -- HIVE-19416 -ALTER TABLE TBLS ADD TXN_ID bigint; ALTER TABLE TBLS ADD WRITE_ID bigint; -ALTER TABLE TBLS ADD WRITEID_LIST CLOB; -ALTER TABLE PARTITIONS ADD TXN_ID bigint; ALTER TABLE PARTITIONS ADD WRITE_ID bigint; -ALTER TABLE PARTITIONS ADD WRITEID_LIST CLOB; -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; diff --git standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql index dde7025a12273e65b19d10bd0c07a292f2450799..3a1f3f3ab6d000c2c2071ac006c845e5a14176d2 100644 --- standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql +++ standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql @@ -164,9 +164,7 @@ CREATE TABLE PARTITIONS PART_NAME VARCHAR2(767) NULL, SD_ID NUMBER NULL, TBL_ID NUMBER NULL, - TXN_ID NUMBER NULL, - WRITE_ID NUMBER NULL, - WRITEID_LIST CLOB NULL + WRITE_ID NUMBER NULL ); ALTER TABLE PARTITIONS ADD CONSTRAINT PARTITIONS_PK PRIMARY KEY (PART_ID); @@ -402,9 +400,7 @@ CREATE TABLE TBLS VIEW_EXPANDED_TEXT CLOB NULL, VIEW_ORIGINAL_TEXT CLOB NULL, IS_REWRITE_ENABLED NUMBER(1) DEFAULT 0 NOT NULL CHECK (IS_REWRITE_ENABLED IN (1,0)), - TXN_ID NUMBER NULL, - WRITE_ID NUMBER NULL, - WRITEID_LIST CLOB NULL + WRITE_ID NUMBER NULL ); ALTER TABLE TBLS ADD CONSTRAINT TBLS_PK PRIMARY KEY (TBL_ID); diff --git standalone-metastore/src/main/sql/oracle/upgrade-3.1.0-to-4.0.0.oracle.sql standalone-metastore/src/main/sql/oracle/upgrade-3.1.0-to-4.0.0.oracle.sql index 0b75831f26fda0f79c5a70d9f833a4c17fa7226d..c94e6ec71c08b678e00f0c64ba1b3d963ec30d69 100644 --- standalone-metastore/src/main/sql/oracle/upgrade-3.1.0-to-4.0.0.oracle.sql +++ standalone-metastore/src/main/sql/oracle/upgrade-3.1.0-to-4.0.0.oracle.sql @@ -1,11 +1,7 @@ SELECT 'Upgrading MetaStore schema from 3.1.0 to 4.0.0' AS Status from dual; -ALTER TABLE TBLS ADD TXN_ID number NULL; ALTER TABLE TBLS ADD WRITE_ID number NULL; -ALTER TABLE TBLS ADD WRITEID_LIST CLOB NULL; -ALTER TABLE PARTITIONS ADD TXN_ID number NULL; ALTER TABLE PARTITIONS ADD WRITE_ID number NULL; -ALTER TABLE PARTITIONS ADD WRITEID_LIST CLOB NULL; -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; diff --git standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql index ad193f9eca4908740a2bea71512f733bc8df0e6f..7ff7d7f6efe2e5002a8ebe236708f2060593b9fd 100644 --- standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql +++ standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql @@ -169,8 +169,7 @@ CREATE TABLE "PARTITIONS" ( "PART_NAME" character varying(767) DEFAULT NULL::character varying, "SD_ID" bigint, "TBL_ID" bigint, - "TXN_ID" bigint, - "WRITEID_LIST" text + "WRITE_ID" bigint DEFAULT 0 ); @@ -395,9 +394,7 @@ CREATE TABLE "TBLS" ( "VIEW_EXPANDED_TEXT" text, "VIEW_ORIGINAL_TEXT" text, "IS_REWRITE_ENABLED" boolean NOT NULL DEFAULT false, - "TXN_ID" bigint, - "WRITE_ID" bigint, - "WRITEID_LIST" text + "WRITE_ID" bigint DEFAULT 0 ); -- diff --git standalone-metastore/src/main/sql/postgres/upgrade-3.1.0-to-4.0.0.postgres.sql standalone-metastore/src/main/sql/postgres/upgrade-3.1.0-to-4.0.0.postgres.sql index d7887197c9ae2d98316fd879132958b4fbcf17ca..eff08b31991a48474fac904bfe14350a7a81a745 100644 --- standalone-metastore/src/main/sql/postgres/upgrade-3.1.0-to-4.0.0.postgres.sql +++ standalone-metastore/src/main/sql/postgres/upgrade-3.1.0-to-4.0.0.postgres.sql @@ -1,12 +1,8 @@ SELECT 'Upgrading MetaStore schema from 3.1.0 to 4.0.0'; -- HIVE-19416 -ALTER TABLE "TBLS" ADD "TXN_ID" bigint; ALTER TABLE "TBLS" ADD "WRITE_ID" bigint; -ALTER TABLE "TBLS" ADD "WRITEID_LIST" text; -ALTER TABLE "PARTITIONS" ADD "TXN_ID" bigint; ALTER TABLE "PARTITIONS" ADD "WRITE_ID" bigint; -ALTER TABLE "PARTITIONS" ADD "WRITEID_LIST" text; -- These lines need to be last. Insert any changes above. UPDATE "VERSION" SET "SCHEMA_VERSION"='4.0.0', "VERSION_COMMENT"='Hive release version 4.0.0' where "VER_ID"=1; diff --git standalone-metastore/src/main/thrift/hive_metastore.thrift standalone-metastore/src/main/thrift/hive_metastore.thrift index 1cf2d17603928bd288f30c77c101ced7a7357940..cc1c5b6543d7da337e70c8b07c10d88a1ec01ef5 100644 --- standalone-metastore/src/main/thrift/hive_metastore.thrift +++ standalone-metastore/src/main/thrift/hive_metastore.thrift @@ -233,12 +233,6 @@ enum SchemaVersionState { DELETED = 8 } -enum IsolationLevelCompliance { - YES = 1, - NO = 2, - UNKNOWN = 3 -} - struct HiveObjectRef{ 1: HiveObjectType objectType, 2: string dbName, @@ -437,10 +431,8 @@ struct Table { 16: optional CreationMetadata creationMetadata, // only for MVs, it stores table names used and txn list at MV creation 17: optional string catName, // Name of the catalog the table is in 18: optional PrincipalType ownerType = PrincipalType.USER, // owner type of this table (default to USER for backward compatibility) - 19: optional i64 txnId=-1, - 20: optional i64 writeId=-1, - 21: optional string validWriteIdList, - 22: optional IsolationLevelCompliance isStatsCompliant + 19: optional i64 writeId=-1, + 20: optional bool isStatsCompliant } struct Partition { @@ -453,10 +445,8 @@ struct Partition { 7: map parameters, 8: optional PrincipalPrivilegeSet privileges, 9: optional string catName, - 10: optional i64 txnId=-1, - 11: optional i64 writeId=-1, - 12: optional string validWriteIdList, - 13: optional IsolationLevelCompliance isStatsCompliant + 10: optional i64 writeId=-1, + 11: optional bool isStatsCompliant } struct PartitionWithoutSD { @@ -484,10 +474,8 @@ struct PartitionSpec { 4: optional PartitionSpecWithSharedSD sharedSDPartitionSpec, 5: optional PartitionListComposingSpec partitionList, 6: optional string catName, - 7: optional i64 txnId=-1, - 8: optional i64 writeId=-1, - 9: optional string validWriteIdList, - 10: optional IsolationLevelCompliance isStatsCompliant + 7: optional i64 writeId=-1, + 8: optional bool isStatsCompliant } // column statistics @@ -583,16 +571,16 @@ struct ColumnStatisticsDesc { struct ColumnStatistics { 1: required ColumnStatisticsDesc statsDesc, 2: required list statsObj, -3: optional i64 txnId=-1, // transaction id of the query that sends this structure +3: optional i64 txnId=-1, // transaction id of the query that sends this structure TODO## needed? 4: optional string validWriteIdList, // valid write id list for the table for which this struct is being sent -5: optional IsolationLevelCompliance isStatsCompliant // Are the stats isolation-level-compliant with the +5: optional bool isStatsCompliant // Are the stats isolation-level-compliant with the // the calling query? } struct AggrStats { 1: required list colStats, 2: required i64 partsFound, // number of partitions for which stats were found -3: optional IsolationLevelCompliance isStatsCompliant +3: optional bool isStatsCompliant } struct SetPartitionsStatsRequest { @@ -730,12 +718,12 @@ struct PartitionsByExprRequest { struct TableStatsResult { 1: required list tableStats, - 2: optional IsolationLevelCompliance isStatsCompliant + 2: optional bool isStatsCompliant } struct PartitionsStatsResult { 1: required map> partStats, - 2: optional IsolationLevelCompliance isStatsCompliant + 2: optional bool isStatsCompliant } struct TableStatsRequest { @@ -760,7 +748,7 @@ struct PartitionsStatsRequest { // Return type for add_partitions_req struct AddPartitionsResult { 1: optional list partitions, - 2: optional IsolationLevelCompliance isStatsCompliant + 2: optional bool isStatsCompliant } // Request type for add_partitions_req @@ -1251,7 +1239,7 @@ struct GetTableRequest { struct GetTableResult { 1: required Table table, - 2: optional IsolationLevelCompliance isStatsCompliant + 2: optional bool isStatsCompliant } struct GetTablesRequest { @@ -1926,7 +1914,9 @@ service ThriftHiveMetastore extends fb303.FacebookService void alter_partitions(1:string db_name, 2:string tbl_name, 3:list new_parts) throws (1:InvalidOperationException o1, 2:MetaException o2) - AlterPartitionsResponse alter_partitions_with_environment_context(1:AlterPartitionsRequest req) + void alter_partitions_with_environment_context(1:string db_name, 2:string tbl_name, 3:list new_parts, 4:EnvironmentContext environment_context) throws (1:InvalidOperationException o1, 2:MetaException o2) + + AlterPartitionsResponse alter_partitions_with_environment_context_req(1:AlterPartitionsRequest req) throws (1:InvalidOperationException o1, 2:MetaException o2) void alter_partition_with_environment_context(1:string db_name, diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index 70771cd477abfc4afd355851b6a1e14f25c58bed..8f4ba421217217f341d49cfbc87784369a58c6ac 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -291,9 +291,10 @@ public boolean dropPartition(String catName, String dbName, String tableName, Li } @Override - public void alterTable(String catName, String dbName, String name, Table newTable) + public void alterTable(String catName, String dbName, String name, Table newTable, + long queryTxnId, String queryValidWriteIds) throws InvalidObjectException, MetaException { - objectStore.alterTable(catName, dbName, name, newTable); + objectStore.alterTable(catName, dbName, name, newTable, queryTxnId, queryValidWriteIds); } @Override @@ -356,17 +357,16 @@ public PartitionValuesResponse listPartitionValues(String catName, String db_nam @Override public void alterPartition(String catName, String dbName, String tblName, List partVals, - Partition newPart) throws InvalidObjectException, MetaException { - objectStore.alterPartition(catName, dbName, tblName, partVals, newPart); + Partition newPart, long queryTxnId, String queryValidWriteIds) throws InvalidObjectException, MetaException { + objectStore.alterPartition(catName, dbName, tblName, partVals, newPart, queryTxnId, queryValidWriteIds); } @Override public void alterPartitions(String catName, String dbName, String tblName, List> partValsList, List newParts, - long txnId, String writeIdList, long writeId) - throws InvalidObjectException, MetaException { + long writeId, long queryTxnId, String queryValidWriteIds) throws InvalidObjectException, MetaException { objectStore.alterPartitions( - catName, dbName, tblName, partValsList, newParts, txnId, writeIdList, writeId); + catName, dbName, tblName, partValsList, newParts, writeId, queryTxnId, queryValidWriteIds); } @Override diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index e9c8993f5e7c13b43ee458754b3923bb13df09da..28462d5875244486f3a8dcc5626908052964ecdf 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -289,7 +289,7 @@ public boolean dropPartition(String catName, String dbName, String tableName, Li } @Override - public void alterTable(String catName, String dbname, String name, Table newTable) + public void alterTable(String catName, String dbname, String name, Table newTable, long queryTxnId, String queryValidWriteIds) throws InvalidObjectException, MetaException { } @@ -358,15 +358,13 @@ public PartitionValuesResponse listPartitionValues(String catName, String db_nam @Override public void alterPartition(String catName, String db_name, String tbl_name, List part_vals, - Partition new_part) throws InvalidObjectException, MetaException { + Partition new_part, long queryTxnId, String queryValidWriteIds) throws InvalidObjectException, MetaException { } @Override public void alterPartitions(String catName, String db_name, String tbl_name, List> part_vals_list, List new_parts, - long txnId, String writeIdList, long writeId) - throws InvalidObjectException, MetaException { - + long writeId, long queryTxnId, String queryValidWriteIds) throws InvalidObjectException, MetaException { } @Override diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java index 28cb3d9ec3891a93c93189ce9d841ed99286fbe6..0f8afb76bdf171ea56d2a1f90dc8d5dc9a29ca9c 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java @@ -1635,7 +1635,13 @@ public void alter_partitions(String dbName, String tblName, List newP req.setTableName(tblName); req.setPartitions(newParts); req.setEnvironmentContext(environmentContext); - client.alter_partitions_with_environment_context(req); + // TODO: this is ugly... account for ability to pass via EC for the old API. + if (environmentContext != null && environmentContext.isSetProperties() + && environmentContext.getProperties().containsKey(StatsSetupConst.VALID_WRITE_IDS)) { + req.setTxnId(Long.parseLong(environmentContext.getProperties().get(StatsSetupConst.TXN_ID))); + req.setValidWriteIdList(environmentContext.getProperties().get(StatsSetupConst.VALID_WRITE_IDS)); + } + client.alter_partitions_with_environment_context_req(req); } @Override @@ -1650,7 +1656,7 @@ public void alter_partitions(String dbName, String tblName, List newP req.setEnvironmentContext(environmentContext); req.setTxnId(txnId); req.setValidWriteIdList(writeIdList); - client.alter_partitions_with_environment_context(req); + client.alter_partitions_with_environment_context_req(req); } @Override diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java index adc82b0b9c1e5e1a08dcb6bd30726510c2b93f6e..d9dd954f7e791008fec9a460712802541e1e3109 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java @@ -60,7 +60,7 @@ public void testAlterTableAddColNotUpdateStats() throws MetaException, InvalidOb getDefaultCatalog(conf), oldTable.getDbName(), oldTable.getTableName(), Arrays.asList("col1", "col2", "col3")); HiveAlterHandler handler = new HiveAlterHandler(); handler.setConf(conf); - handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable); + handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable, null); } @Test @@ -85,7 +85,7 @@ public void testAlterTableDelColUpdateStats() throws MetaException, InvalidObjec RawStore msdb = Mockito.mock(RawStore.class); HiveAlterHandler handler = new HiveAlterHandler(); handler.setConf(conf); - handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable); + handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable, null); Mockito.verify(msdb, Mockito.times(1)).getTableColumnStatistics( getDefaultCatalog(conf), oldTable.getDbName(), oldTable.getTableName(), Arrays.asList("col1", "col2", "col3", "col4") ); @@ -115,7 +115,7 @@ public void testAlterTableChangePosNotUpdateStats() throws MetaException, Invali getDefaultCatalog(conf), oldTable.getDbName(), oldTable.getTableName(), Arrays.asList("col1", "col2", "col3", "col4")); HiveAlterHandler handler = new HiveAlterHandler(); handler.setConf(conf); - handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable); + handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable, null); } } diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java index 5d5bc76e37e87618c7b8ff0232d600c965b06846..3f5ef40f8787b1ec4efa5cde4623f017629de0f4 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java @@ -245,7 +245,7 @@ public void testTableOps() throws MetaException, InvalidObjectException, NoSuchO newTbl1.setOwner("role1"); newTbl1.setOwnerType(PrincipalType.ROLE); - objectStore.alterTable(DEFAULT_CATALOG_NAME, DB1, TABLE1, newTbl1); + objectStore.alterTable(DEFAULT_CATALOG_NAME, DB1, TABLE1, newTbl1, -1, null); tables = objectStore.getTables(DEFAULT_CATALOG_NAME, DB1, "new*"); Assert.assertEquals(1, tables.size()); Assert.assertEquals("new" + TABLE1, tables.get(0)); diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java index 62ed380dfcc22e90ccdf4b1516cc33f128f6b3e7..7cf5c267bbe5e7ffcc36e0055979d201d454bd27 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java @@ -232,7 +232,7 @@ public void testTableOps() throws Exception { tblOwner = "role1"; tbl.setOwner(tblOwner); tbl.setOwnerType(PrincipalType.ROLE); - objectStore.alterTable(DEFAULT_CATALOG_NAME, dbName, tblName, tbl); + objectStore.alterTable(DEFAULT_CATALOG_NAME, dbName, tblName, tbl, -1, null); tbl = objectStore.getTable(DEFAULT_CATALOG_NAME, dbName, tblName); Assert.assertEquals("Owner of the table did not change.", tblOwner, tbl.getOwner()); @@ -338,7 +338,7 @@ public void testPartitionOps() throws Exception { Partition ptn1Atl = new Partition(Arrays.asList(ptnColVal1Alt), dbName, tblName, 0, 0, tbl.getSd(), partParams); ptn1Atl.setCatName(DEFAULT_CATALOG_NAME); - objectStore.alterPartition(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal1), ptn1Atl); + objectStore.alterPartition(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal1), ptn1Atl, -1, null); ptn1Atl = objectStore.getPartition(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal1Alt)); // Drop an existing partition ("bbb") via ObjectStore diff --git storage-api/src/java/org/apache/hive/common/util/TxnIdUtils.java storage-api/src/java/org/apache/hive/common/util/TxnIdUtils.java index 4b00864a0815640f87ed9bc8bf6727f44bfb607c..4b3cb7d85ec4600200c6d98c0e41028ca2ac6f4a 100644 --- storage-api/src/java/org/apache/hive/common/util/TxnIdUtils.java +++ storage-api/src/java/org/apache/hive/common/util/TxnIdUtils.java @@ -43,37 +43,6 @@ public static boolean checkEquivalentWriteIds(ValidWriteIdList a, ValidWriteIdLi } /** - * Check if the give two write id lists are for concurrent writes - * on the table. - */ - public static boolean areTheseConcurrentWrites( - ValidWriteIdList older, long olderWriteId, - ValidWriteIdList newer, long newerWriteId) { - if (!older.getTableName().equalsIgnoreCase(newer.getTableName())) { - return false; - } - - assert(older.getHighWatermark() <= newer.getHighWatermark()); - - // Return false when a write id is not positive. - if (olderWriteId <= 0 || newerWriteId <= 0) { - return false; - } - - // If olderWriteId is for aborted write, return false. - if (newer.isWriteIdAborted(olderWriteId)) { - return false; - } - - // TODO: does this need to account for watermark? - // If either writeId is contained in the other's writeIdList, - // it is a concurrent INSERTs case. - int index2Older = Arrays.binarySearch(older.getInvalidWriteIds(), olderWriteId); - int index2Newer = Arrays.binarySearch(newer.getInvalidWriteIds(), newerWriteId); - return index2Older >= 0 || index2Newer >= 0; - } - - /** * Check the min open ID/highwater mark/exceptions list to see if 2 ID lists are at the same commit point. * This can also be used for ValidTxnList as well as ValidWriteIdList. */