diff --git standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkTool.java standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkTool.java index 041cd76234..006bf65d5f 100644 --- standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkTool.java +++ standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkTool.java @@ -69,7 +69,7 @@ private String tableName = TEST_TABLE + "_" + System.getProperty("user.name"); - @Option(names = {"-N", "--number"}, description = "umber of object instances") + @Option(names = {"-N", "--number"}, description = "number of object instances") private int[] instances = {100}; @Option(names = {"-L", "--spin"}, description = "spin count") @@ -198,7 +198,9 @@ public void run() { .add("renameTable" + '.' + howMany, () -> benchmarkRenameTable(bench, bData, howMany)) .add("dropDatabase" + '.' + howMany, - () -> benchmarkDropDatabase(bench, bData, howMany)); + () -> benchmarkDropDatabase(bench, bData, howMany)) + .add("lock", + () -> benchmarkLocks(bench, bData, howMany)); } if (doList) { diff --git standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSBenchmarks.java standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSBenchmarks.java index f53f2ef43b..97ff0d6231 100644 --- standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSBenchmarks.java +++ standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSBenchmarks.java @@ -20,31 +20,16 @@ import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; import org.apache.hadoop.hive.metastore.TableType; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.*; import org.apache.thrift.TException; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.URI; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; +import java.util.*; import java.util.stream.IntStream; -import static org.apache.hadoop.hive.metastore.tools.Util.addManyPartitions; -import static org.apache.hadoop.hive.metastore.tools.Util.addManyPartitionsNoException; -import static org.apache.hadoop.hive.metastore.tools.Util.createSchema; -import static org.apache.hadoop.hive.metastore.tools.Util.generatePartitionNames; -import static java.util.concurrent.Executors.newFixedThreadPool; -import static org.apache.hadoop.hive.metastore.tools.Util.throwingSupplierWrapper; +import static org.apache.hadoop.hive.metastore.tools.Util.*; /** * Actual benchmark code. @@ -408,6 +393,45 @@ static DescriptiveStatistics benchmarkDropDatabase(@NotNull MicroBenchmark bench } } + static DescriptiveStatistics benchmarkLocks(@NotNull MicroBenchmark bench, + @NotNull BenchData data, + int nTables){ + HMSClient client = data.getClient(); + String dbName = data.dbName; + String tableNameFormat = "tmp_table_%d"; + String user = "hclient"; + List lockComponents = new ArrayList<>(); + List txnIds = new ArrayList<>(); + + try { + LOG.info("Beginning prep"); + // 1. create nTables number of tables + LOG.info("Creating {} tables", nTables); + createManyTables(client, nTables, dbName, tableNameFormat); + // 2. create LockComponents + LOG.info("Creating LockComponents for tables"); + for (int i = 0; i < nTables; i++) { + lockComponents.add(new Util.LockComponentBuilder() + .setDbName(dbName) + .setTableName(String.format(tableNameFormat, i)) + .setShared() + .setOperationType(DataOperationType.SELECT) + .build()); + } + + return bench.measure(() -> { + long txnId = executeOpenTxn(client, user); + txnIds.add(txnId); + executeLock(client, txnId, user, lockComponents); + }); + } finally { + LOG.info("Cleanup"); + txnIds.forEach(txnId -> executeCommitTxn(client, txnId)); + txnIds.forEach(txnId -> executeUnlock(client, txnId)); + dropManyTables(client, nTables, dbName, tableNameFormat); + } + } + private static void createManyTables(HMSClient client, int howMany, String dbName, String format) { List columns = createSchema(new ArrayList<>(Arrays.asList("name", "string"))); List partitions = createSchema(new ArrayList<>(Arrays.asList("date", "string"))); @@ -437,6 +461,33 @@ private static void createPartitionedTable(HMSClient client, String dbName, Stri .build())); } + private static void executeLock(HMSClient client, long txnId, String user, List lockComponents) { + LOG.debug("execute lock"); + throwingSupplierWrapper(() -> client.lock( + new Util.LockRequestBuilder(Thread.currentThread().getName()) + .setUser(user) + .setTransactionId(txnId) + .addLockComponents(lockComponents) + .build() + )); + } + + private static void executeUnlock(HMSClient client, long lockId) { + throwingSupplierWrapper(() -> client.unlock(new UnlockRequest(lockId))); + } + + private static long executeOpenTxn(HMSClient client, String user) { + return throwingSupplierWrapper(() -> client.openTxn(user)); + } + + private static void executeAbortTxns(HMSClient client, List txnIds) { + throwingSupplierWrapper(() -> client.abortTxns(txnIds)); + } + + private static void executeCommitTxn(HMSClient client, long txnId) { + throwingSupplierWrapper(() -> client.commitTxn(txnId)); + } + static DescriptiveStatistics benchmarkGetNotificationId(@NotNull MicroBenchmark benchmark, @NotNull BenchData data) { HMSClient client = data.getClient(); diff --git standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSClient.java standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSClient.java index 7cc1e42a8b..b3f8e71fb3 100644 --- standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSClient.java +++ standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSClient.java @@ -20,14 +20,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.DropPartitionsRequest; -import org.apache.hadoop.hive.metastore.api.DropPartitionsResult; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.RequestPartsSpec; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore; +import org.apache.hadoop.hive.metastore.api.*; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; @@ -48,9 +41,7 @@ import javax.security.auth.login.LoginException; import java.io.File; import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; +import java.net.*; import java.security.PrivilegedExceptionAction; import java.util.Arrays; import java.util.HashSet; @@ -319,6 +310,39 @@ void appendPartition(@NotNull String dbName, @NotNull String tableName, client.append_partition_with_environment_context(dbName, tableName, partitionValues, null); } + + LockResponse lock(@NotNull LockRequest rqst) throws TException { + return client.lock(rqst); + } + + LockResponse checkLock(@NotNull CheckLockRequest rqst) throws TException { + return client.check_lock(rqst); + } + + boolean unlock(@NotNull UnlockRequest rqst) throws TException { + client.unlock(rqst); + return true; + } + + GetOpenTxnsResponse getOpenTxns() throws TException { + return client.get_open_txns(); + } + + boolean commitTxn(long txnId) throws TException { + client.commit_txn(new CommitTxnRequest(txnId)); + return true; + } + + boolean abortTxns(List txnIds) throws TException { + client.abort_txns(new AbortTxnsRequest(txnIds)); + return true; + } + + long openTxn(String user) throws TException { + OpenTxnsResponse txns = openTxnsIntr(user, 1, null, null, null); + return txns.getTxn_ids().get(0); + } + private TTransport open(Configuration conf, @NotNull URI uri) throws TException, IOException, LoginException { boolean useSSL = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.USE_SSL); @@ -418,6 +442,31 @@ private TTransport open(Configuration conf, @NotNull URI uri) throws return transport; } + private OpenTxnsResponse openTxnsIntr(String user, int numTxns, String replPolicy, + List srcTxnIds, TxnType txnType) throws TException { + String hostname; + try { + hostname = InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + LOG.error("Unable to resolve my host name " + e.getMessage()); + throw new RuntimeException(e); + } + OpenTxnRequest rqst = new OpenTxnRequest(numTxns, user, hostname); + if (replPolicy != null) { + assert srcTxnIds != null; + assert numTxns == srcTxnIds.size(); + // need to set this only for replication tasks + rqst.setReplPolicy(replPolicy); + rqst.setReplSrcTxnIds(srcTxnIds); + } else { + assert srcTxnIds == null; + } + if (txnType != null) { + rqst.setTxn_type(txnType); + } + return client.open_txns(rqst); + } + @Override public void close() throws Exception { if (transport != null && transport.isOpen()) { diff --git standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/Util.java standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/Util.java index 101d6759c5..ae58efa805 100644 --- standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/Util.java +++ standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/Util.java @@ -21,27 +21,18 @@ import com.google.common.base.Joiner; import com.google.common.net.HostAndPort; import org.apache.hadoop.hive.metastore.TableType; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.PrincipalType; -import org.apache.hadoop.hive.metastore.api.SerDeInfo; -import org.apache.hadoop.hive.metastore.api.StorageDescriptor; -import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.*; import org.apache.thrift.TException; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.net.InetAddress; import java.net.URI; import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.net.UnknownHostException; +import java.util.*; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -383,6 +374,229 @@ Partition build() { } } + /** + * Builder of lock requests + */ + public static class LockRequestBuilder { + private LockRequest req; + private LockTrie trie; + private boolean userSet; + + LockRequestBuilder(String agentInfo) { + req = new LockRequest(); + trie = new LockTrie(); + userSet = false; + + if(agentInfo != null) { + req.setAgentInfo(agentInfo); + } + } + + public LockRequest build() { + if (!userSet) { + throw new RuntimeException("Cannot build a lock without giving a user"); + } + trie.addLocksToRequest(req); + try { + req.setHostname(InetAddress.getLocalHost().getHostName()); + } catch (UnknownHostException e) { + throw new RuntimeException("Unable to determine our local host!"); + } + return req; + } + + /** + * Set the transaction id. + * @param txnid transaction id + * @return reference to this builder + */ + public LockRequestBuilder setTransactionId(long txnid) { + req.setTxnid(txnid); + return this; + } + + public LockRequestBuilder setUser(String user) { + if (user == null) user = "unknown"; + req.setUser(user); + userSet = true; + return this; + } + + /** + * Add a lock component to the lock request + * @param component to add + * @return reference to this builder + */ + public LockRequestBuilder addLockComponent(LockComponent component) { + trie.add(component); + return this; + } + + /** + * Add a collection with lock components to the lock request + * @param components to add + * @return reference to this builder + */ + public LockRequestBuilder addLockComponents(Collection components) { + trie.addAll(components); + return this; + } + + private static class LockTrie { + Map trie; + + LockTrie() { + trie = new LinkedHashMap<>(); + } + + public void add(LockComponent comp) { + TableTrie tabs = trie.get(comp.getDbname()); + if (tabs == null) { + tabs = new TableTrie(); + trie.put(comp.getDbname(), tabs); + } + setTable(comp, tabs); + } + + public void addAll(Collection components) { + for(LockComponent component: components) { + add(component); + } + } + + public void addLocksToRequest(LockRequest request) { + for (TableTrie tab : trie.values()) { + for (PartTrie part : tab.values()) { + for (LockComponent lock : part.values()) { + request.addToComponent(lock); + } + } + } + } + + private void setTable(LockComponent comp, TableTrie tabs) { + PartTrie parts = tabs.get(comp.getTablename()); + if (parts == null) { + parts = new PartTrie(); + tabs.put(comp.getTablename(), parts); + } + setPart(comp, parts); + } + + private void setPart(LockComponent comp, PartTrie parts) { + LockComponent existing = parts.get(comp.getPartitionname()); + if (existing == null) { + // No existing lock for this partition. + parts.put(comp.getPartitionname(), comp); + } else if (existing.getType() != LockType.EXCLUSIVE && (comp.getType() == LockType.EXCLUSIVE + || comp.getType() == LockType.SHARED_WRITE)) { + // We only need to promote if comp.type is > existing.type. For + // efficiency we check if existing is exclusive (in which case we + // need never promote) or if comp is exclusive or shared_write (in + // which case we can promote even though they may both be shared + // write). If comp is shared_read there's never a need to promote. + parts.put(comp.getPartitionname(), comp); + } + } + } + + private static class TableTrie extends LinkedHashMap {} + private static class PartTrie extends LinkedHashMap {} + } + + public static class LockComponentBuilder { + private LockComponent component; + private boolean tableNameSet; + private boolean partNameSet; + + public LockComponentBuilder() { + component = new LockComponent(); + tableNameSet = partNameSet = false; + } + + /** + * Set the lock to be exclusive. + * @return reference to this builder + */ + public LockComponentBuilder setExclusive() { + component.setType(LockType.EXCLUSIVE); + return this; + } + + /** + * Set the lock to be semi-shared. + * @return reference to this builder + */ + public LockComponentBuilder setSemiShared() { + component.setType(LockType.SHARED_WRITE); + return this; + } + + /** + * Set the lock to be shared. + * @return reference to this builder + */ + public LockComponentBuilder setShared() { + component.setType(LockType.SHARED_READ); + return this; + } + + /** + * Set the database name. + * @param dbName database name + * @return reference to this builder + */ + public LockComponentBuilder setDbName(String dbName) { + component.setDbname(dbName); + return this; + } + + public LockComponentBuilder setIsTransactional(boolean t) { + component.setIsTransactional(t); + return this; + } + + public LockComponentBuilder setOperationType(DataOperationType dop) { + component.setOperationType(dop); + return this; + } + + /** + * Set the table name. + * @param tableName table name + * @return reference to this builder + */ + public LockComponentBuilder setTableName(String tableName) { + component.setTablename(tableName); + tableNameSet = true; + return this; + } + + /** + * Set the partition name. + * @param partitionName partition name + * @return reference to this builder + */ + public LockComponentBuilder setPartitionName(String partitionName) { + component.setPartitionname(partitionName); + partNameSet = true; + return this; + } + + public LockComponent build() { + LockLevel level = LockLevel.DB; + if (tableNameSet) level = LockLevel.TABLE; + if (partNameSet) level = LockLevel.PARTITION; + component.setLevel(level); + return component; + } + + public LockComponent setLock(LockType type) { + component.setType(type); + return component; + } + } + /** * Create table schema from parameters. *