From fd79fa7c244b6c538864cb6a0a1d006b039b416b Mon Sep 17 00:00:00 2001 From: Alexander Kolbasov Date: Thu, 23 Aug 2018 18:10:53 -0700 Subject: [PATCH 1/1] HIVE-20388: Move common classes out of metastore-server --- .../hadoop/hive/ql/lockmgr/DbTxnManager.java | 7 +- .../hadoop/hive/ql/stats/StatsUpdaterThread.java | 3 +- .../hive/ql/util/HiveStrictManagedMigration.java | 3 +- .../apache/hive/service/auth/HiveAuthFactory.java | 10 +- .../apache/hadoop/hive/common/StatsSetupConst.java | 0 .../apache/hadoop/hive/metastore/ColumnType.java | 0 .../hadoop/hive/metastore/HiveMetaStoreClient.java | 8 +- .../hadoop/hive/metastore/IMetaStoreClient.java | 0 .../apache/hadoop/hive/metastore/MetaStoreFS.java | 0 .../hadoop/hive/metastore/ReplChangeManager.java | 0 .../hadoop/hive/metastore/TableIterable.java | 0 .../apache/hadoop/hive/metastore/Warehouse.java | 0 .../hadoop/hive/metastore/conf/MetastoreConf.java | 0 .../hadoop/hive/metastore/conf/TimeValidator.java | 0 .../spec/CompositePartitionSpecProxy.java | 0 .../spec/PartitionListComposingSpecProxy.java | 0 .../partition/spec/PartitionSpecProxy.java | 0 .../spec/PartitionSpecWithSharedSDProxy.java | 0 .../security/DelegationTokenIdentifier.java | 0 .../security/DelegationTokenSecretManager.java | 0 .../security/DelegationTokenSelector.java | 0 .../metastore/security/HadoopThriftAuthBridge.java | 0 .../security/HadoopThriftAuthBridge23.java | 0 .../hadoop/hive/metastore/txn/TxnCommonUtils.java | 127 +++++++++++++++++++++ .../hadoop/hive/metastore/utils/FileUtils.java | 0 .../hadoop/hive/metastore/utils/HdfsUtils.java | 0 .../hive/metastore/utils/MetaStoreUtils.java | 46 +------- .../hadoop/hive/metastore/utils/SecurityUtils.java | 53 +-------- .../hadoop/hive/metastore/HiveAlterHandler.java | 8 +- .../hadoop/hive/metastore/HiveMetaStore.java | 4 +- .../MaterializationsRebuildLockCleanerTask.java | 3 +- .../security/MetastoreDelegationTokenManager.java | 6 +- .../hadoop/hive/metastore/txn/TxnHandler.java | 2 +- .../apache/hadoop/hive/metastore/txn/TxnUtils.java | 104 +---------------- .../hive/metastore/utils/MetaStoreServerUtils.java | 85 ++++++++++++-- .../metastore/HiveMetaStoreClientPreCatalog.java | 8 +- 36 files changed, 246 insertions(+), 231 deletions(-) rename standalone-metastore/{metastore-server => metastore-common}/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java (100%) rename standalone-metastore/{metastore-server => metastore-common}/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java (100%) rename standalone-metastore/{metastore-server => metastore-common}/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (99%) rename standalone-metastore/{metastore-server => metastore-common}/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java (100%) rename standalone-metastore/{metastore-server => metastore-common}/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java (100%) rename standalone-metastore/{metastore-server => metastore-common}/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java (100%) rename standalone-metastore/{metastore-server => metastore-common}/src/main/java/org/apache/hadoop/hive/metastore/TableIterable.java (100%) rename standalone-metastore/{metastore-server => metastore-common}/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java (100%) rename standalone-metastore/{metastore-server => metastore-common}/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java (100%) rename standalone-metastore/{metastore-server => metastore-common}/src/main/java/org/apache/hadoop/hive/metastore/conf/TimeValidator.java (100%) rename standalone-metastore/{metastore-server => metastore-common}/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java (100%) rename standalone-metastore/{metastore-server => metastore-common}/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionListComposingSpecProxy.java (100%) rename standalone-metastore/{metastore-server => metastore-common}/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecProxy.java (100%) rename standalone-metastore/{metastore-server => metastore-common}/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecWithSharedSDProxy.java (100%) rename standalone-metastore/{metastore-server => metastore-common}/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenIdentifier.java (100%) rename standalone-metastore/{metastore-server => metastore-common}/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenSecretManager.java (100%) rename standalone-metastore/{metastore-server => metastore-common}/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenSelector.java (100%) rename standalone-metastore/{metastore-server => metastore-common}/src/main/java/org/apache/hadoop/hive/metastore/security/HadoopThriftAuthBridge.java (100%) rename standalone-metastore/{metastore-server => metastore-common}/src/main/java/org/apache/hadoop/hive/metastore/security/HadoopThriftAuthBridge23.java (100%) create mode 100644 standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnCommonUtils.java rename standalone-metastore/{metastore-server => metastore-common}/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java (100%) rename standalone-metastore/{metastore-server => metastore-common}/src/main/java/org/apache/hadoop/hive/metastore/utils/HdfsUtils.java (100%) rename standalone-metastore/{metastore-server => metastore-common}/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java (95%) rename standalone-metastore/{metastore-server => metastore-common}/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java (85%) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java index f746d2dff3..4e3d74c715 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java @@ -18,13 +18,11 @@ Licensed to the Apache Software Foundation (ASF) under one package org.apache.hadoop.hive.ql.lockmgr; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.common.ValidTxnWriteIdList; -import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.LockRequestBuilder; @@ -37,7 +35,7 @@ Licensed to the Apache Software Foundation (ASF) under one import org.apache.hadoop.hive.metastore.api.TxnAbortedException; import org.apache.hadoop.hive.metastore.api.TxnToWriteId; import org.apache.hadoop.hive.metastore.api.CommitTxnRequest; -import org.apache.hadoop.hive.metastore.txn.TxnUtils; +import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryPlan; @@ -50,7 +48,6 @@ Licensed to the Apache Software Foundation (ASF) under one import org.apache.hadoop.hive.ql.plan.LockTableDesc; import org.apache.hadoop.hive.ql.plan.UnlockDatabaseDesc; import org.apache.hadoop.hive.ql.plan.UnlockTableDesc; -import org.apache.hadoop.hive.ql.session.*; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hive.common.util.ShutdownHookManager; import org.apache.thrift.TException; @@ -676,7 +673,7 @@ public ValidTxnWriteIdList getValidWriteIds(List tableList, assert isTxnOpen(); assert validTxnList != null && !validTxnList.isEmpty(); try { - return TxnUtils.createValidTxnWriteIdList( + return TxnCommonUtils.createValidTxnWriteIdList( txnId, getMS().getValidWriteIds(tableList, validTxnList)); } catch (TException e) { throw new LockException(ErrorMsg.METASTORE_COMMUNICATION_FAILED.getMsg(), e); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUpdaterThread.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUpdaterThread.java index a50ec18b8a..981f6aed5f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUpdaterThread.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUpdaterThread.java @@ -52,6 +52,7 @@ import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.StatsUpdateMode; +import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils; import org.apache.hadoop.hive.metastore.txn.TxnStore; import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.ql.DriverUtils; @@ -500,7 +501,7 @@ private ValidReaderWriteIdList getWriteIds( // TODO: acid utils don't support catalogs GetValidWriteIdsRequest req = new GetValidWriteIdsRequest( Lists.newArrayList(fullTableName.getDbTable()), null); - return TxnUtils.createValidReaderWriteIdList( + return TxnCommonUtils.createValidReaderWriteIdList( txnHandler.getValidWriteIds(req).getTblValidWriteIds().get(0)); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java b/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java index 54df541c66..13a8af7ce7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java @@ -54,6 +54,7 @@ import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.TransactionalValidationListener; import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils; import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.metastore.utils.HiveStrictManagedUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; @@ -994,7 +995,7 @@ private TxnCtx generateTxnCtxForAlter( String fqn = table.getDbName() + "." + table.getTableName(); List writeIdsObj = msc.getValidWriteIds( Lists.newArrayList(fqn), txns.toString()); - String validWriteIds = TxnUtils.createValidTxnWriteIdList(txnId, writeIdsObj) + String validWriteIds = TxnCommonUtils.createValidTxnWriteIdList(txnId, writeIdsObj) .getTableValidWriteIdList(fqn).writeToString(); long writeId = msc.allocateTableWriteId(txnId, table.getDbName(), table.getTableName()); result = new TxnCtx(writeId, validWriteIds, txnId); diff --git a/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java b/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java index d23e4389a7..f5f6d49f6b 100644 --- a/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java +++ b/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java @@ -27,21 +27,15 @@ import javax.security.sasl.AuthenticationException; import javax.security.sasl.Sasl; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.conf.HiveConfUtil; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.shims.HadoopShims.KerberosNameShim; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.metastore.security.DBTokenStore; -import org.apache.hadoop.hive.metastore.security.DelegationTokenStore; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; -import org.apache.hadoop.hive.metastore.security.MemoryTokenStore; import org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager; -import org.apache.hadoop.hive.metastore.security.ZooKeeperTokenStore; -import org.apache.hadoop.hive.metastore.utils.SecurityUtils; import org.apache.hadoop.security.SaslRpcServer.AuthMethod; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; @@ -97,7 +91,7 @@ public HiveAuthFactory(HiveConf conf) throws TTransportException { delegationTokenManager = new MetastoreDelegationTokenManager(); try { Object baseHandler = null; - String tokenStoreClass = SecurityUtils.getTokenStoreClassName(conf); + String tokenStoreClass = MetaStoreServerUtils.getTokenStoreClassName(conf); if (tokenStoreClass.equals(DBTokenStore.class.getName())) { // IMetaStoreClient is needed to access token store if DBTokenStore is to be used. It diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java similarity index 100% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java rename to standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java similarity index 100% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java rename to standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java similarity index 99% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java rename to standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index faf6810420..a0ff79cc5c 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -62,7 +62,7 @@ import org.apache.hadoop.hive.metastore.hooks.URIResolverHook; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; -import org.apache.hadoop.hive.metastore.txn.TxnUtils; +import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils; import org.apache.hadoop.hive.metastore.utils.JavaUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.metastore.utils.ObjectPair; @@ -2726,19 +2726,19 @@ public boolean removeMasterKey(Integer keySeq) throws TException { @Override public ValidTxnList getValidTxns() throws TException { - return TxnUtils.createValidReadTxnList(client.get_open_txns(), 0); + return TxnCommonUtils.createValidReadTxnList(client.get_open_txns(), 0); } @Override public ValidTxnList getValidTxns(long currentTxn) throws TException { - return TxnUtils.createValidReadTxnList(client.get_open_txns(), currentTxn); + return TxnCommonUtils.createValidReadTxnList(client.get_open_txns(), currentTxn); } @Override public ValidWriteIdList getValidWriteIds(String fullTableName) throws TException { GetValidWriteIdsRequest rqst = new GetValidWriteIdsRequest(Collections.singletonList(fullTableName), null); GetValidWriteIdsResponse validWriteIds = client.get_valid_write_ids(rqst); - return TxnUtils.createValidReaderWriteIdList(validWriteIds.getTblValidWriteIds().get(0)); + return TxnCommonUtils.createValidReaderWriteIdList(validWriteIds.getTblValidWriteIds().get(0)); } @Override diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java similarity index 100% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java rename to standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java similarity index 100% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java rename to standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java similarity index 100% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java rename to standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TableIterable.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TableIterable.java similarity index 100% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TableIterable.java rename to standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TableIterable.java diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java similarity index 100% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java rename to standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java similarity index 100% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java rename to standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/TimeValidator.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/TimeValidator.java similarity index 100% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/TimeValidator.java rename to standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/TimeValidator.java diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java similarity index 100% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java rename to standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionListComposingSpecProxy.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionListComposingSpecProxy.java similarity index 100% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionListComposingSpecProxy.java rename to standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionListComposingSpecProxy.java diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecProxy.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecProxy.java similarity index 100% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecProxy.java rename to standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecProxy.java diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecWithSharedSDProxy.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecWithSharedSDProxy.java similarity index 100% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecWithSharedSDProxy.java rename to standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecWithSharedSDProxy.java diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenIdentifier.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenIdentifier.java similarity index 100% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenIdentifier.java rename to standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenIdentifier.java diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenSecretManager.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenSecretManager.java similarity index 100% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenSecretManager.java rename to standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenSecretManager.java diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenSelector.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenSelector.java similarity index 100% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenSelector.java rename to standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenSelector.java diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/HadoopThriftAuthBridge.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/HadoopThriftAuthBridge.java similarity index 100% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/HadoopThriftAuthBridge.java rename to standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/HadoopThriftAuthBridge.java diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/HadoopThriftAuthBridge23.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/HadoopThriftAuthBridge23.java similarity index 100% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/HadoopThriftAuthBridge23.java rename to standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/HadoopThriftAuthBridge23.java diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnCommonUtils.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnCommonUtils.java new file mode 100644 index 0000000000..94cb18dccc --- /dev/null +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnCommonUtils.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.txn; + +import org.apache.hadoop.hive.common.ValidReadTxnList; +import org.apache.hadoop.hive.common.ValidReaderWriteIdList; +import org.apache.hadoop.hive.common.ValidTxnList; +import org.apache.hadoop.hive.common.ValidTxnWriteIdList; +import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse; +import org.apache.hadoop.hive.metastore.api.TableValidWriteIds; + +import java.util.BitSet; +import java.util.Collections; +import java.util.List; + +public class TxnCommonUtils { + /** + * Transform a {@link org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse} to a + * {@link org.apache.hadoop.hive.common.ValidTxnList}. This assumes that the caller intends to + * read the files, and thus treats both open and aborted transactions as invalid. + * @param txns txn list from the metastore + * @param currentTxn Current transaction that the user has open. If this is greater than 0 it + * will be removed from the exceptions list so that the user sees his own + * transaction as valid. + * @return a valid txn list. + */ + public static ValidTxnList createValidReadTxnList(GetOpenTxnsResponse txns, long currentTxn) { + /* + * The highWaterMark should be min(currentTxn,txns.getTxn_high_water_mark()) assuming currentTxn>0 + * otherwise if currentTxn=7 and 8 commits before 7, then 7 will see result of 8 which + * doesn't make sense for Snapshot Isolation. Of course for Read Committed, the list should + * include the latest committed set. + */ + long highWaterMark = (currentTxn > 0) ? Math.min(currentTxn, txns.getTxn_high_water_mark()) + : txns.getTxn_high_water_mark(); + + // Open txns are already sorted in ascending order. This list may or may not include HWM + // but it is guaranteed that list won't have txn > HWM. But, if we overwrite the HWM with currentTxn + // then need to truncate the exceptions list accordingly. + List openTxns = txns.getOpen_txns(); + + // We care only about open/aborted txns below currentTxn and hence the size should be determined + // for the exceptions list. The currentTxn will be missing in openTxns list only in rare case like + // txn is aborted by AcidHouseKeeperService and compactor actually cleans up the aborted txns. + // So, for such cases, we get negative value for sizeToHwm with found position for currentTxn, and so, + // we just negate it to get the size. + int sizeToHwm = (currentTxn > 0) ? Collections.binarySearch(openTxns, currentTxn) : openTxns.size(); + sizeToHwm = (sizeToHwm < 0) ? (-sizeToHwm) : sizeToHwm; + long[] exceptions = new long[sizeToHwm]; + BitSet inAbortedBits = BitSet.valueOf(txns.getAbortedBits()); + BitSet outAbortedBits = new BitSet(); + long minOpenTxnId = Long.MAX_VALUE; + int i = 0; + for (long txn : openTxns) { + // For snapshot isolation, we don't care about txns greater than current txn and so stop here. + // Also, we need not include current txn to exceptions list. + if ((currentTxn > 0) && (txn >= currentTxn)) { + break; + } + if (inAbortedBits.get(i)) { + outAbortedBits.set(i); + } else if (minOpenTxnId == Long.MAX_VALUE) { + minOpenTxnId = txn; + } + exceptions[i++] = txn; + } + return new ValidReadTxnList(exceptions, outAbortedBits, highWaterMark, minOpenTxnId); + } + + /** + * Transform a {@link org.apache.hadoop.hive.metastore.api.GetValidWriteIdsResponse} to a + * {@link org.apache.hadoop.hive.common.ValidTxnWriteIdList}. This assumes that the caller intends to + * read the files, and thus treats both open and aborted transactions as invalid. + * @param currentTxnId current txn ID for which we get the valid write ids list + * @param list valid write ids list from the metastore + * @return a valid write IDs list for the whole transaction. + */ + public static ValidTxnWriteIdList createValidTxnWriteIdList(Long currentTxnId, + List validIds) { + ValidTxnWriteIdList validTxnWriteIdList = new ValidTxnWriteIdList(currentTxnId); + for (TableValidWriteIds tableWriteIds : validIds) { + validTxnWriteIdList.addTableValidWriteIdList(createValidReaderWriteIdList(tableWriteIds)); + } + return validTxnWriteIdList; + } + + /** + * Transform a {@link TableValidWriteIds} to a + * {@link org.apache.hadoop.hive.common.ValidReaderWriteIdList}. This assumes that the caller intends to + * read the files, and thus treats both open and aborted write ids as invalid. + * @param tableWriteIds valid write ids for the given table from the metastore + * @return a valid write IDs list for the input table + */ + public static ValidReaderWriteIdList createValidReaderWriteIdList(TableValidWriteIds tableWriteIds) { + String fullTableName = tableWriteIds.getFullTableName(); + long highWater = tableWriteIds.getWriteIdHighWaterMark(); + List invalids = tableWriteIds.getInvalidWriteIds(); + BitSet abortedBits = BitSet.valueOf(tableWriteIds.getAbortedBits()); + long[] exceptions = new long[invalids.size()]; + int i = 0; + for (long writeId : invalids) { + exceptions[i++] = writeId; + } + if (tableWriteIds.isSetMinOpenWriteId()) { + return new ValidReaderWriteIdList(fullTableName, exceptions, abortedBits, highWater, + tableWriteIds.getMinOpenWriteId()); + } else { + return new ValidReaderWriteIdList(fullTableName, exceptions, abortedBits, highWater); + } + } +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java similarity index 100% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java rename to standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/HdfsUtils.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/HdfsUtils.java similarity index 100% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/HdfsUtils.java rename to standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/HdfsUtils.java diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java similarity index 95% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java rename to standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java index 5233bee592..c681a87a1c 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java @@ -245,46 +245,6 @@ public static boolean isPropertyTrue(Map tableParams, String pro return "TRUE".equalsIgnoreCase(tableParams.get(prop)); } - // check if stats need to be (re)calculated - public static boolean requireCalStats(Partition oldPart, - Partition newPart, Table tbl, - EnvironmentContext environmentContext) { - - if (environmentContext != null - && environmentContext.isSetProperties() - && StatsSetupConst.TRUE.equals(environmentContext.getProperties().get( - StatsSetupConst.DO_NOT_UPDATE_STATS))) { - return false; - } - - if (MetaStoreServerUtils.isView(tbl)) { - return false; - } - - if (oldPart == null && newPart == null) { - return true; - } - - // requires to calculate stats if new partition doesn't have it - if ((newPart == null) || (newPart.getParameters() == null) - || !MetaStoreServerUtils.containsAllFastStats(newPart.getParameters())) { - return true; - } - - if (environmentContext != null && environmentContext.isSetProperties()) { - String statsType = environmentContext.getProperties().get(StatsSetupConst.STATS_GENERATED); - // no matter STATS_GENERATED is USER or TASK, all need to re-calculate the stats: - // USER: alter table .. update statistics - // TASK: from some sql operation which could collect and compute stats - if (StatsSetupConst.TASK.equals(statsType) || StatsSetupConst.USER.equals(statsType)) { - return true; - } - } - - // requires to calculate stats if new and old have different fast stats - return !MetaStoreServerUtils.isFastStatsSame(oldPart, newPart); - } - /** Duplicates AcidUtils; used in a couple places in metastore. */ public static boolean isInsertOnlyTableParam(Map params) { @@ -939,4 +899,10 @@ public static String getDefaultCatalog(Configuration conf) { return catName; } + public static boolean isView(Table table) { + if (table == null) { + return false; + } + return TableType.VIRTUAL_VIEW.toString().equals(table.getTableType()); + } } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java similarity index 85% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java rename to standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java index 0b0cfbda80..bae1ec35b2 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java @@ -17,13 +17,8 @@ */ package org.apache.hadoop.hive.metastore.utils; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.security.DBTokenStore; import org.apache.hadoop.hive.metastore.security.DelegationTokenIdentifier; import org.apache.hadoop.hive.metastore.security.DelegationTokenSelector; -import org.apache.hadoop.hive.metastore.security.MemoryTokenStore; -import org.apache.hadoop.hive.metastore.security.ZooKeeperTokenStore; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; @@ -31,31 +26,29 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.security.token.TokenSelector; -import org.apache.zookeeper.client.ZooKeeperSaslClient; - -import javax.security.auth.login.AppConfigurationEntry; import org.apache.thrift.transport.TSSLTransportFactory; import org.apache.thrift.transport.TServerSocket; import org.apache.thrift.transport.TSocket; import org.apache.thrift.transport.TTransport; import org.apache.thrift.transport.TTransportException; +import org.apache.zookeeper.client.ZooKeeperSaslClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.net.ssl.SSLParameters; import javax.net.ssl.SSLServerSocket; import javax.net.ssl.SSLSocket; -import javax.security.auth.login.LoginException; +import javax.security.auth.login.AppConfigurationEntry; import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag; - +import javax.security.auth.login.LoginException; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; public class SecurityUtils { private static final Logger LOG = LoggerFactory.getLogger(SecurityUtils.class); @@ -192,42 +185,6 @@ public static void setTokenStr(UserGroupInformation ugi, String tokenStr, String return delegationToken; } - private static final String DELEGATION_TOKEN_STORE_CLS = "hive.cluster.delegation.token.store.class"; - - /** - * This method should be used to return the metastore specific tokenstore class name to main - * backwards compatibility - * - * @param conf - HiveConf object - * @return the tokenStoreClass name from the HiveConf. It maps the hive specific tokenstoreclass - * name to metastore module specific class name. For eg: - * hive.cluster.delegation.token.store.class is set to - * org.apache.hadoop.hive.thrift.MemoryTokenStore it returns the equivalent tokenstore - * class defined in the metastore module which is - * org.apache.hadoop.hive.metastore.security.MemoryTokenStore Similarly, - * org.apache.hadoop.hive.thrift.DBTokenStore maps to - * org.apache.hadoop.hive.metastore.security.DBTokenStore and - * org.apache.hadoop.hive.thrift.ZooKeeperTokenStore maps to - * org.apache.hadoop.hive.metastore.security.ZooKeeperTokenStore - */ - public static String getTokenStoreClassName(Configuration conf) { - String tokenStoreClass = conf.get(DELEGATION_TOKEN_STORE_CLS, ""); - if (StringUtils.isBlank(tokenStoreClass)) { - // default tokenstore is MemoryTokenStore - return MemoryTokenStore.class.getName(); - } - switch (tokenStoreClass) { - case "org.apache.hadoop.hive.thrift.DBTokenStore": - return DBTokenStore.class.getName(); - case "org.apache.hadoop.hive.thrift.MemoryTokenStore": - return MemoryTokenStore.class.getName(); - case "org.apache.hadoop.hive.thrift.ZooKeeperTokenStore": - return ZooKeeperTokenStore.class.getName(); - default: - return tokenStoreClass; - } - } - /** * @return the user name set in hadoop.job.ugi param or the current user from System diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index c551b80d8a..6a1cd3bc9a 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -316,7 +316,7 @@ public void alterTable(RawStore msdb, Warehouse wh, String catName, String dbnam } } else { // operations other than table rename - if (MetaStoreUtils.requireCalStats(null, null, newt, environmentContext) && + if (MetaStoreServerUtils.requireCalStats(null, null, newt, environmentContext) && !isPartitionedTable) { Database db = msdb.getDatabase(catName, newDbName); // Update table stats. For partitioned table, we update stats in alterPartition() @@ -467,7 +467,7 @@ public Partition alterPartition(RawStore msdb, Warehouse wh, String catName, Str "Unable to alter partition because table or database does not exist."); } oldPart = msdb.getPartition(catName, dbname, name, new_part.getValues()); - if (MetaStoreUtils.requireCalStats(oldPart, new_part, tbl, environmentContext)) { + if (MetaStoreServerUtils.requireCalStats(oldPart, new_part, tbl, environmentContext)) { // if stats are same, no need to update if (MetaStoreServerUtils.isFastStatsSame(oldPart, new_part)) { MetaStoreServerUtils.updateBasicState(environmentContext, new_part.getParameters()); @@ -611,7 +611,7 @@ public Partition alterPartition(RawStore msdb, Warehouse wh, String catName, Str new_part.getSd().setLocation(oldPart.getSd().getLocation()); } - if (MetaStoreUtils.requireCalStats(oldPart, new_part, tbl, environmentContext)) { + if (MetaStoreServerUtils.requireCalStats(oldPart, new_part, tbl, environmentContext)) { MetaStoreServerUtils.updatePartitionStatsFast( new_part, tbl, wh, false, true, environmentContext, false); } @@ -711,7 +711,7 @@ public Partition alterPartition(RawStore msdb, Warehouse wh, String catName, Str oldParts.add(oldTmpPart); partValsList.add(tmpPart.getValues()); - if (MetaStoreUtils.requireCalStats(oldTmpPart, tmpPart, tbl, environmentContext)) { + if (MetaStoreServerUtils.requireCalStats(oldTmpPart, tmpPart, tbl, environmentContext)) { // Check if stats are same, no need to update if (MetaStoreServerUtils.isFastStatsSame(oldTmpPart, tmpPart)) { MetaStoreServerUtils.updateBasicState(environmentContext, tmpPart.getParameters()); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 6b660f55ac..33b22a9fc3 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -1862,7 +1862,7 @@ private void create_table_core(final RawStore ms, final Table tbl, } } if (MetastoreConf.getBoolVar(conf, ConfVars.STATS_AUTO_GATHER) && - !MetaStoreServerUtils.isView(tbl)) { + !MetaStoreUtils.isView(tbl)) { MetaStoreServerUtils.updateTableStatsSlow(db, tbl, wh, madeDir, false, envContext); } @@ -3804,7 +3804,7 @@ private boolean canUpdateStats(Table tbl) { updateStatsTbl = !Boolean.valueOf(tblParams.get(StatsSetupConst.DO_NOT_UPDATE_STATS)); } if (!MetastoreConf.getBoolVar(conf, ConfVars.STATS_AUTO_GATHER) || - MetaStoreServerUtils.isView(tbl) || + MetaStoreUtils.isView(tbl) || !updateStatsTbl) { return false; } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockCleanerTask.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockCleanerTask.java index 9ce7d6db49..d35c9602a6 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockCleanerTask.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockCleanerTask.java @@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils; import org.apache.hadoop.hive.metastore.txn.TxnStore; import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.slf4j.Logger; @@ -62,7 +63,7 @@ public void run() { TxnStore.MutexAPI.LockHandle handle = null; try { handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.MaterializationRebuild.name()); - ValidTxnList validTxnList = TxnUtils.createValidReadTxnList(txnHandler.getOpenTxns(), 0); + ValidTxnList validTxnList = TxnCommonUtils.createValidReadTxnList(txnHandler.getOpenTxns(), 0); long removedCnt = txnHandler.cleanupMaterializationRebuildLocks(validTxnList, MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.TXN_TIMEOUT, TimeUnit.MILLISECONDS)); if (removedCnt > 0) { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java index 8d84fd7f32..b35dc7ce4b 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java @@ -20,13 +20,11 @@ package org.apache.hadoop.hive.metastore.security; import java.io.IOException; -import java.security.PrivilegedExceptionAction; import java.util.concurrent.TimeUnit; -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; -import org.apache.hadoop.hive.metastore.utils.SecurityUtils; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.UserGroupInformation; @@ -139,7 +137,7 @@ public String verifyDelegationToken(String tokenStrForm) throws IOException { } private DelegationTokenStore getTokenStore(Configuration conf) throws IOException { - String tokenStoreClassName = SecurityUtils.getTokenStoreClassName(conf); + String tokenStoreClassName = MetaStoreServerUtils.getTokenStoreClassName(conf); try { Class storeClass = Class.forName(tokenStoreClassName).asSubclass(DelegationTokenStore.class); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index 015dfd9ec9..2df61b41ea 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -1252,7 +1252,7 @@ public GetValidWriteIdsResponse getValidWriteIds(GetValidWriteIdsRequest rqst) validTxnList = new ValidReadTxnList(rqst.getValidTxnList()); } else { // Passing 0 for currentTxn means, this validTxnList is not wrt to any txn - validTxnList = TxnUtils.createValidReadTxnList(getOpenTxns(), 0); + validTxnList = TxnCommonUtils.createValidReadTxnList(getOpenTxns(), 0); } try { /** diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java index bd202edb91..3bb1f0c62c 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java @@ -20,19 +20,16 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.ValidCompactorWriteIdList; import org.apache.hadoop.hive.common.ValidReaderWriteIdList; -import org.apache.hadoop.hive.common.ValidReadTxnList; -import org.apache.hadoop.hive.common.ValidTxnList; -import org.apache.hadoop.hive.common.ValidTxnWriteIdList; -import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.metastore.TransactionalValidationListener; -import org.apache.hadoop.hive.metastore.api.*; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.TableValidWriteIds; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hadoop.hive.metastore.utils.JavaUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collections; import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; @@ -48,101 +45,6 @@ static final public char STAT_COMMITTED = 'c'; static final public char STAT_OBSOLETE = 's'; - /** - * Transform a {@link org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse} to a - * {@link org.apache.hadoop.hive.common.ValidTxnList}. This assumes that the caller intends to - * read the files, and thus treats both open and aborted transactions as invalid. - * @param txns txn list from the metastore - * @param currentTxn Current transaction that the user has open. If this is greater than 0 it - * will be removed from the exceptions list so that the user sees his own - * transaction as valid. - * @return a valid txn list. - */ - public static ValidTxnList createValidReadTxnList(GetOpenTxnsResponse txns, long currentTxn) { - /* - * The highWaterMark should be min(currentTxn,txns.getTxn_high_water_mark()) assuming currentTxn>0 - * otherwise if currentTxn=7 and 8 commits before 7, then 7 will see result of 8 which - * doesn't make sense for Snapshot Isolation. Of course for Read Committed, the list should - * include the latest committed set. - */ - long highWaterMark = (currentTxn > 0) ? Math.min(currentTxn, txns.getTxn_high_water_mark()) - : txns.getTxn_high_water_mark(); - - // Open txns are already sorted in ascending order. This list may or may not include HWM - // but it is guaranteed that list won't have txn > HWM. But, if we overwrite the HWM with currentTxn - // then need to truncate the exceptions list accordingly. - List openTxns = txns.getOpen_txns(); - - // We care only about open/aborted txns below currentTxn and hence the size should be determined - // for the exceptions list. The currentTxn will be missing in openTxns list only in rare case like - // txn is aborted by AcidHouseKeeperService and compactor actually cleans up the aborted txns. - // So, for such cases, we get negative value for sizeToHwm with found position for currentTxn, and so, - // we just negate it to get the size. - int sizeToHwm = (currentTxn > 0) ? Collections.binarySearch(openTxns, currentTxn) : openTxns.size(); - sizeToHwm = (sizeToHwm < 0) ? (-sizeToHwm) : sizeToHwm; - long[] exceptions = new long[sizeToHwm]; - BitSet inAbortedBits = BitSet.valueOf(txns.getAbortedBits()); - BitSet outAbortedBits = new BitSet(); - long minOpenTxnId = Long.MAX_VALUE; - int i = 0; - for (long txn : openTxns) { - // For snapshot isolation, we don't care about txns greater than current txn and so stop here. - // Also, we need not include current txn to exceptions list. - if ((currentTxn > 0) && (txn >= currentTxn)) { - break; - } - if (inAbortedBits.get(i)) { - outAbortedBits.set(i); - } else if (minOpenTxnId == Long.MAX_VALUE) { - minOpenTxnId = txn; - } - exceptions[i++] = txn; - } - return new ValidReadTxnList(exceptions, outAbortedBits, highWaterMark, minOpenTxnId); - } - - /** - * Transform a {@link org.apache.hadoop.hive.metastore.api.GetValidWriteIdsResponse} to a - * {@link org.apache.hadoop.hive.common.ValidTxnWriteIdList}. This assumes that the caller intends to - * read the files, and thus treats both open and aborted transactions as invalid. - * @param currentTxnId current txn ID for which we get the valid write ids list - * @param list valid write ids list from the metastore - * @return a valid write IDs list for the whole transaction. - */ - public static ValidTxnWriteIdList createValidTxnWriteIdList(Long currentTxnId, - List validIds) { - ValidTxnWriteIdList validTxnWriteIdList = new ValidTxnWriteIdList(currentTxnId); - for (TableValidWriteIds tableWriteIds : validIds) { - validTxnWriteIdList.addTableValidWriteIdList(createValidReaderWriteIdList(tableWriteIds)); - } - return validTxnWriteIdList; - } - - /** - * Transform a {@link org.apache.hadoop.hive.metastore.api.TableValidWriteIds} to a - * {@link org.apache.hadoop.hive.common.ValidReaderWriteIdList}. This assumes that the caller intends to - * read the files, and thus treats both open and aborted write ids as invalid. - * @param tableWriteIds valid write ids for the given table from the metastore - * @return a valid write IDs list for the input table - */ - public static ValidReaderWriteIdList createValidReaderWriteIdList(TableValidWriteIds tableWriteIds) { - String fullTableName = tableWriteIds.getFullTableName(); - long highWater = tableWriteIds.getWriteIdHighWaterMark(); - List invalids = tableWriteIds.getInvalidWriteIds(); - BitSet abortedBits = BitSet.valueOf(tableWriteIds.getAbortedBits()); - long[] exceptions = new long[invalids.size()]; - int i = 0; - for (long writeId : invalids) { - exceptions[i++] = writeId; - } - if (tableWriteIds.isSetMinOpenWriteId()) { - return new ValidReaderWriteIdList(fullTableName, exceptions, abortedBits, highWater, - tableWriteIds.getMinOpenWriteId()); - } else { - return new ValidReaderWriteIdList(fullTableName, exceptions, abortedBits, highWater); - } - } - /** * Transform a {@link org.apache.hadoop.hive.metastore.api.TableValidWriteIds} to a * {@link org.apache.hadoop.hive.common.ValidCompactorWriteIdList}. This assumes that the caller intends to diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java index 506d9ab559..10ff9dfbb6 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java @@ -79,7 +79,10 @@ import org.apache.hadoop.hive.metastore.columnstats.merge.ColumnStatsMergerFactory; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; +import org.apache.hadoop.hive.metastore.security.DBTokenStore; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; +import org.apache.hadoop.hive.metastore.security.MemoryTokenStore; +import org.apache.hadoop.hive.metastore.security.ZooKeeperTokenStore; import org.apache.hadoop.security.authorize.DefaultImpersonationProvider; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.util.MachineList; @@ -105,6 +108,7 @@ public String apply(@Nullable String string) { return org.apache.commons.lang.StringUtils.defaultString(string); } }; + private static final String DELEGATION_TOKEN_STORE_CLS = "hive.cluster.delegation.token.store.class"; /** * We have a need to sanity-check the map before conversion from persisted objects to @@ -347,13 +351,6 @@ public static boolean validateColumnName(String name) { return true; } - public static boolean isView(Table table) { - if (table == null) { - return false; - } - return TableType.VIRTUAL_VIEW.toString().equals(table.getTableType()); - } - /** * @param partParams * @return True if the passed Parameters Map contains values for all "Fast Stats". @@ -906,6 +903,80 @@ private static boolean isValidTypeChar(char c) { return Character.isLetterOrDigit(c) || c == '_'; } + // check if stats need to be (re)calculated + public static boolean requireCalStats(Partition oldPart, + Partition newPart, Table tbl, + EnvironmentContext environmentContext) { + + if (environmentContext != null + && environmentContext.isSetProperties() + && StatsSetupConst.TRUE.equals(environmentContext.getProperties().get( + StatsSetupConst.DO_NOT_UPDATE_STATS))) { + return false; + } + + if (MetaStoreUtils.isView(tbl)) { + return false; + } + + if (oldPart == null && newPart == null) { + return true; + } + + // requires to calculate stats if new partition doesn't have it + if ((newPart == null) || (newPart.getParameters() == null) + || !containsAllFastStats(newPart.getParameters())) { + return true; + } + + if (environmentContext != null && environmentContext.isSetProperties()) { + String statsType = environmentContext.getProperties().get(StatsSetupConst.STATS_GENERATED); + // no matter STATS_GENERATED is USER or TASK, all need to re-calculate the stats: + // USER: alter table .. update statistics + // TASK: from some sql operation which could collect and compute stats + if (StatsSetupConst.TASK.equals(statsType) || StatsSetupConst.USER.equals(statsType)) { + return true; + } + } + + // requires to calculate stats if new and old have different fast stats + return !isFastStatsSame(oldPart, newPart); + } + + /** + * This method should be used to return the metastore specific tokenstore class name to main + * backwards compatibility + * + * @param conf - HiveConf object + * @return the tokenStoreClass name from the HiveConf. It maps the hive specific tokenstoreclass + * name to metastore module specific class name. For eg: + * hive.cluster.delegation.token.store.class is set to + * org.apache.hadoop.hive.thrift.MemoryTokenStore it returns the equivalent tokenstore + * class defined in the metastore module which is + * org.apache.hadoop.hive.metastore.security.MemoryTokenStore Similarly, + * org.apache.hadoop.hive.thrift.DBTokenStore maps to + * org.apache.hadoop.hive.metastore.security.DBTokenStore and + * org.apache.hadoop.hive.thrift.ZooKeeperTokenStore maps to + * org.apache.hadoop.hive.metastore.security.ZooKeeperTokenStore + */ + public static String getTokenStoreClassName(Configuration conf) { + String tokenStoreClass = conf.get(DELEGATION_TOKEN_STORE_CLS, ""); + if (StringUtils.isBlank(tokenStoreClass)) { + // default tokenstore is MemoryTokenStore + return MemoryTokenStore.class.getName(); + } + switch (tokenStoreClass) { + case "org.apache.hadoop.hive.thrift.DBTokenStore": + return DBTokenStore.class.getName(); + case "org.apache.hadoop.hive.thrift.MemoryTokenStore": + return MemoryTokenStore.class.getName(); + case "org.apache.hadoop.hive.thrift.ZooKeeperTokenStore": + return ZooKeeperTokenStore.class.getName(); + default: + return tokenStoreClass; + } + } + // ColumnStatisticsObj with info about its db, table, partition (if table is partitioned) public static class ColStatsObjWithSourceInfo { private final ColumnStatisticsObj colStatsObj; diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java index 6e0d9c1848..4093aa7a18 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java @@ -62,7 +62,7 @@ import org.apache.hadoop.hive.metastore.hooks.URIResolverHook; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; -import org.apache.hadoop.hive.metastore.txn.TxnUtils; +import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils; import org.apache.hadoop.hive.metastore.utils.JavaUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.metastore.utils.ObjectPair; @@ -2187,19 +2187,19 @@ public boolean removeMasterKey(Integer keySeq) throws TException { @Override public ValidTxnList getValidTxns() throws TException { - return TxnUtils.createValidReadTxnList(client.get_open_txns(), 0); + return TxnCommonUtils.createValidReadTxnList(client.get_open_txns(), 0); } @Override public ValidTxnList getValidTxns(long currentTxn) throws TException { - return TxnUtils.createValidReadTxnList(client.get_open_txns(), currentTxn); + return TxnCommonUtils.createValidReadTxnList(client.get_open_txns(), currentTxn); } @Override public ValidWriteIdList getValidWriteIds(String fullTableName) throws TException { GetValidWriteIdsRequest rqst = new GetValidWriteIdsRequest(Collections.singletonList(fullTableName), null); GetValidWriteIdsResponse validWriteIds = client.get_valid_write_ids(rqst); - return TxnUtils.createValidReaderWriteIdList(validWriteIds.getTblValidWriteIds().get(0)); + return TxnCommonUtils.createValidReaderWriteIdList(validWriteIds.getTblValidWriteIds().get(0)); } @Override -- 2.16.3