diff --git standalone-metastore/metastore-tools/metastore-benchmarks/pom.xml standalone-metastore/metastore-tools/metastore-benchmarks/pom.xml index 612ff6fd74..599e8c51a4 100644 --- standalone-metastore/metastore-tools/metastore-benchmarks/pom.xml +++ standalone-metastore/metastore-tools/metastore-benchmarks/pom.xml @@ -29,6 +29,7 @@ ../.. + 1.23 @@ -74,6 +75,16 @@ + + org.openjdk.jmh + jmh-core + ${jmh.version} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + org.slf4j diff --git standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/ACIDBenchmarks.java standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/ACIDBenchmarks.java new file mode 100644 index 0000000000..2ebd9ba0a2 --- /dev/null +++ standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/ACIDBenchmarks.java @@ -0,0 +1,248 @@ +package org.apache.hadoop.hive.metastore.tools; + +import org.apache.hadoop.hive.metastore.api.DataOperationType; +import org.apache.hadoop.hive.metastore.api.LockComponent; +import org.apache.hadoop.hive.metastore.api.LockRequest; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.thrift.TException; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +import static org.apache.hadoop.hive.metastore.tools.BenchmarkUtils.createManyTables; +import static org.apache.hadoop.hive.metastore.tools.BenchmarkUtils.dropManyTables; +import static org.apache.hadoop.hive.metastore.tools.Util.throwingSupplierWrapper; + +public class ACIDBenchmarks { + + private static final Logger LOG = LoggerFactory.getLogger(CoreContext.class); + + @State(Scope.Benchmark) + public static class CoreContext { + @Param("1") + protected int howMany; + + @State(Scope.Thread) + public static class ThreadState { + HMSClient client; + + @Setup + public void doSetup() throws Exception { + LOG.debug("Creating client"); + client = HMSConfig.getInstance().newClient(); + } + + @TearDown + public void doTearDown() throws Exception { + client.close(); + LOG.debug("Closed a connection to metastore."); + } + } + + @Setup + public void setup() { + LoggerContext ctx = (LoggerContext) LogManager.getContext(false); + Configuration ctxConfig = ctx.getConfiguration(); + ctxConfig.getLoggerConfig(CoreContext.class.getName()).setLevel(Level.INFO); + ctx.updateLoggers(ctxConfig); + } + } + + @State(Scope.Benchmark) + public static class TestOpenTxn extends CoreContext { + + @State(Scope.Thread) + public static class ThreadState extends CoreContext.ThreadState { + List openTxns = new ArrayList<>(); + + @TearDown + public void doTearDown() throws Exception { + client.abortTxns(openTxns); + LOG.debug("aborted all opened txns"); + } + + void addTxn(List openTxn) { + openTxns.addAll(openTxn); + } + } + + @Benchmark + public void openTxn(TestOpenTxn.ThreadState state) throws TException { + state.addTxn(state.client.openTxn(howMany)); + LOG.debug("opened txns, count=", howMany); + } + } + + @State(Scope.Benchmark) + public static class TestLocking extends CoreContext { + private int nTables; + + @Param("0") + private int nPartitions; + + private List lockComponents; + + @Setup + public void setup() { + this.nTables = (nPartitions != 0) ? howMany / nPartitions : howMany; + createLockComponents(); + } + + @State(Scope.Thread) + public static class ThreadState extends CoreContext.ThreadState { + List openTxns = new ArrayList<>(); + long txnId; + + @Setup(org.openjdk.jmh.annotations.Level.Invocation) + public void iterSetup() { + txnId = executeOpenTxnAndGetTxnId(client); + LOG.debug("opened txn, id={}", txnId); + openTxns.add(txnId); + } + + @TearDown + public void doTearDown() throws Exception { + client.abortTxns(openTxns); + if (BenchmarkUtils.checkTxnsCleaned(client, openTxns) == false) { + LOG.error("Something went wrong with the cleanup of txns"); + } + LOG.debug("aborted all opened txns"); + } + } + + @Benchmark + public void lock(TestLocking.ThreadState state) { + LOG.debug("sending lock request"); + executeLock(state.client, state.txnId, lockComponents); + } + + private void createLockComponents() { + lockComponents = new ArrayList<>(); + + for (int i = 0; i < nTables; i++) { + for (int j = 0; j < nPartitions - (nPartitions > 1 ? 1 : 0); j++) { + lockComponents.add( + new Util.LockComponentBuilder() + .setDbName("default") + .setTableName(String.format("tmp_table_%d", i)) + .setPartitionName("p_" + j) + .setShared() + .setOperationType(DataOperationType.SELECT) + .build()); + } + if (nPartitions != 1) { + lockComponents.add( + new Util.LockComponentBuilder() + .setDbName("default") + .setTableName(String.format("tmp_table_%d", i)) + .setShared() + .setOperationType(DataOperationType.SELECT) + .build()); + } + } + } + + private static long executeOpenTxnAndGetTxnId(HMSClient client) { + return throwingSupplierWrapper(() -> client.openTxn(1).get(0)); + } + + private void executeLock(HMSClient client, long txnId, List lockComponents) { + LockRequest req = new LockRequest(lockComponents, "hclient", "localhost"); + req.setTxnid(txnId); + throwingSupplierWrapper(() -> client.lock(req)); + } + } + + @State(Scope.Benchmark) + public static class TestAllocateTableWriteIds extends CoreContext { + String dbName = "test_db"; + String tblName = "tmp_table"; + + @State(Scope.Thread) + public static class ThreadState extends CoreContext.ThreadState { + List openTxns = new ArrayList<>(); + long txnId; + + @Setup + public void iterSetup() { + txnId = executeOpenTxnAndGetTxnId(client); + LOG.info("opened txn, id={}", txnId); + openTxns.add(txnId); + } + + @TearDown + public void doTearDown() throws Exception { + client.abortTxns(openTxns); + if (BenchmarkUtils.checkTxnsCleaned(client, openTxns) == false) { + LOG.error("Something went wrong with the cleanup of txns"); + } + LOG.info("aborted all opened txns"); + } + } + + @Benchmark + public void allocateTableWriteIds(TestAllocateTableWriteIds.ThreadState state) throws TException { + state.client.allocateTableWriteIds(dbName, tblName, state.openTxns); + } + + private static long executeOpenTxnAndGetTxnId(HMSClient client) { + return throwingSupplierWrapper(() -> client.openTxn(1).get(0)); + } + } + + @State(Scope.Benchmark) + public static class TestGetValidWriteIds extends CoreContext { + String dbName = "test_db"; + String tblName = "table_%d"; + List fullTableNames = new ArrayList<>(); + HMSClient client; + + @Setup + public void doSetup() { + try { + client = HMSConfig.getInstance().newClient(); + } catch (Exception e) { + LOG.error(e.getMessage()); + } + + try { + if (!client.dbExists(dbName)) { + client.createDatabase(dbName); + } + } catch (TException e) { + LOG.error(e.getMessage()); + } + + LOG.info("creating {} tables", this.howMany); + createManyTables(client, this.howMany, dbName, tblName); + for (int i = 0; i < this.howMany; i++) { + fullTableNames.add(dbName + ".table_" + i); + } + } + + @TearDown + public void doTearDown() throws Exception { + LOG.debug("dropping {} tables", howMany); + dropManyTables(client, howMany, dbName, tblName); + } + + + @Benchmark + public void getValidWriteIds(TestGetValidWriteIds.ThreadState state) throws TException { + LOG.debug("executing getValidWriteIds"); + state.client.getValidWriteIds(this.fullTableNames); + } + } +} + diff --git standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkTool.java standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkTool.java index 2ab9388301..2211970832 100644 --- standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkTool.java +++ standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkTool.java @@ -19,6 +19,12 @@ package org.apache.hadoop.hive.metastore.tools; import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.ChainedOptionsBuilder; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine; @@ -31,12 +37,32 @@ import java.nio.file.Paths; import java.util.Arrays; import java.util.Formatter; +import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; import static org.apache.hadoop.hive.metastore.tools.Constants.HMS_DEFAULT_PORT; -import static org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.*; +import static org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkCreatePartition; +import static org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkCreatePartitions; +import static org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkDeleteCreate; +import static org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkDeleteWithPartitions; +import static org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkDropDatabase; +import static org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkDropPartition; +import static org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkDropPartitions; +import static org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkGetNotificationId; +import static org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkGetPartitionNames; +import static org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkGetPartitions; +import static org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkGetPartitionsByName; +import static org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkGetTable; +import static org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkListAllTables; +import static org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkListDatabases; +import static org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkListManyPartitions; +import static org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkListPartition; +import static org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkListTables; +import static org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkOpenTxns; +import static org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkRenameTable; +import static org.apache.hadoop.hive.metastore.tools.HMSBenchmarks.benchmarkTableCreate; import static org.apache.hadoop.hive.metastore.tools.Util.getServerUri; import static picocli.CommandLine.Command; import static picocli.CommandLine.Option; @@ -54,6 +80,11 @@ private static final TimeUnit scale = TimeUnit.MILLISECONDS; private static final String CSV_SEPARATOR = "\t"; private static final String TEST_TABLE = "bench_table"; + private enum RunModes { + ACID, + NONACID, + ALL + } @Option(names = {"-H", "--host"}, description = "HMS Host", paramLabel = "URI") @@ -68,7 +99,6 @@ @Option(names = {"-t", "--table"}, description = "table name") private String tableName = TEST_TABLE + "_" + System.getProperty("user.name"); - @Option(names = {"-N", "--number"}, description = "number of object instances") private int[] instances = {100}; @@ -84,7 +114,7 @@ @Option(names = {"-o", "--output"}, description = "output file") private String outputFile; - @Option(names = {"-T", "--threads"}, description = "number of concurrent threads") + @Option(names = {"-T", "--threads"}, description = "number of concurrent threads/clients") private int nThreads = 2; @Option(names = {"--confdir"}, description = "configuration directory") @@ -97,7 +127,7 @@ private boolean doCSV = false; @Option(names = {"--params"}, description = "number of table/partition parameters") - private int nParameters = 0; + private int[] nParameters = {0}; @Option(names = {"--savedata"}, description = "save raw data in specified dir") private String dataSaveDir; @@ -111,6 +141,10 @@ @Option(names = {"-E", "--exclude"}, description = "test name patterns to exclude") private Pattern[] exclude; + @Option(names = {"--runMode"}, + description = "flag for setting the mode for the benchmark, acceptable values are: ACID, NONACID, ALL") + private RunModes runMode = RunModes.ALL; + public static void main(String[] args) { CommandLine.run(new BenchmarkTool(), args); } @@ -141,12 +175,62 @@ private static void saveDataFile(String location, String name, } } - @Override public void run() { - LOG.info("Using warmup " + warmup + - " spin " + spinCount + " nparams " + nParameters + " threads " + nThreads); + LOG.info("Using warmup " + warmup + " spin " + spinCount + " nparams " + Arrays.toString(nParameters) + " threads " + + nThreads); + HMSConfig.getInstance().init(host, port, confDir); + + if (runMode == RunModes.ALL) { + runAcidBenchmarks(); + runNonAcidBenchmarks(); + } else if (runMode == RunModes.ACID) { + runAcidBenchmarks(); + } else { + runNonAcidBenchmarks(); + } + } + private void runAcidBenchmarks() { + ChainedOptionsBuilder optsBuilder = + new OptionsBuilder() + .warmupIterations(warmup) + .measurementIterations(spinCount) + .operationsPerInvocation(1) + .mode(Mode.SingleShotTime) + .timeUnit(TimeUnit.MILLISECONDS) + .forks(0) + .threads(nThreads) + .syncIterations(true); + + String[] candidates = new String[] { + ACIDBenchmarks.TestOpenTxn.class.getSimpleName(), + ACIDBenchmarks.TestLocking.class.getSimpleName(), + ACIDBenchmarks.TestGetValidWriteIds.class.getSimpleName(), + ACIDBenchmarks.TestAllocateTableWriteIds.class.getSimpleName() + }; + + for (String pattern : Util.filterMatches(Arrays.asList(candidates), matches, exclude)) { + optsBuilder = optsBuilder.include(pattern); + } + + Options opts = + optsBuilder + .param("howMany", Arrays.stream(instances) + .mapToObj(String::valueOf) + .toArray(String[]::new)) + .param("nPartitions", Arrays.stream(nParameters) + .mapToObj(String::valueOf).toArray(String[]::new)) + .build(); + + try { + new Runner(opts).run(); + } catch (RunnerException e) { + LOG.error(e.getMessage(), e); + } + } + + private void runNonAcidBenchmarks() { StringBuilder sb = new StringBuilder(); BenchData bData = new BenchData(dbName, tableName); @@ -163,7 +247,7 @@ public void run() { .add("createTable", () -> benchmarkTableCreate(bench, bData)) .add("dropTable", () -> benchmarkDeleteCreate(bench, bData)) .add("dropTableWithPartitions", - () -> benchmarkDeleteWithPartitions(bench, bData, 1, nParameters)) + () -> benchmarkDeleteWithPartitions(bench, bData, 1, nParameters[0])) .add("addPartition", () -> benchmarkCreatePartition(bench, bData)) .add("dropPartition", () -> benchmarkDropPartition(bench, bData)) .add("listPartition", () -> benchmarkListPartition(bench, bData)) @@ -184,7 +268,7 @@ public void run() { suite.add("listTables" + '.' + howMany, () -> benchmarkListTables(bench, bData, howMany)) .add("dropTableWithPartitions" + '.' + howMany, - () -> benchmarkDeleteWithPartitions(bench, bData, howMany, nParameters)) + () -> benchmarkDeleteWithPartitions(bench, bData, howMany, nParameters[0])) .add("listPartitions" + '.' + howMany, () -> benchmarkListManyPartitions(bench, bData, howMany)) .add("getPartitions" + '.' + howMany, @@ -205,8 +289,12 @@ public void run() { () -> benchmarkOpenTxns(bench, bData, howMany)); } + List toRun = suite.listMatching(matches, exclude); + if (toRun.isEmpty()) { + return; + } if (doList) { - suite.listMatching(matches, exclude).forEach(System.out::println); + toRun.forEach(System.out::println); return; } @@ -253,7 +341,7 @@ public void run() { saveData(result.getResult(), dataSaveDir, scale); } } catch (Exception e) { - e.printStackTrace(); + LOG.error(e.getMessage(), e); } } } diff --git standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkUtils.java standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkUtils.java new file mode 100644 index 0000000000..ecacc78d82 --- /dev/null +++ standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkUtils.java @@ -0,0 +1,72 @@ +package org.apache.hadoop.hive.metastore.tools; + +import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.TxnInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.IntStream; + +import static org.apache.hadoop.hive.metastore.tools.Util.createSchema; +import static org.apache.hadoop.hive.metastore.tools.Util.throwingSupplierWrapper; + +public class BenchmarkUtils { + private static final Logger LOG = LoggerFactory.getLogger(BenchmarkUtils.class); + + + static void createManyTables(HMSClient client, int howMany, String dbName, String format) { + List columns = createSchema(new ArrayList<>(Arrays.asList("name", "string"))); + List partitions = createSchema(new ArrayList<>(Arrays.asList("date", "string"))); + IntStream.range(0, howMany) + .forEach(i -> + throwingSupplierWrapper(() -> client.createTable( + new Util.TableBuilder(dbName, String.format(format, i)) + .withType(TableType.MANAGED_TABLE) + .withColumns(columns) + .withPartitionKeys(partitions) + .build()))); + } + + static void dropManyTables(HMSClient client, int howMany, String dbName, String format) { + IntStream.range(0, howMany) + .forEach(i -> + throwingSupplierWrapper(() -> client.dropTable(dbName, String.format(format, i)))); + } + + // Create a simple table with a single column and single partition + static void createPartitionedTable(HMSClient client, String dbName, String tableName) { + throwingSupplierWrapper(() -> client.createTable( + new Util.TableBuilder(dbName, tableName) + .withType(TableType.MANAGED_TABLE) + .withColumns(createSchema(Collections.singletonList("name:string"))) + .withPartitionKeys(createSchema(Collections.singletonList("date"))) + .build())); + } + + static boolean checkTxnsCleaned(HMSClient client, List txnsOpenedByBenchmark) throws InterruptedException { + // let's wait the default cleaner run period + Thread.sleep(100000); + List notCleanedTxns = new ArrayList<>(); + throwingSupplierWrapper(() -> { + List txnInfos = client.getOpenTxnsInfo(); + txnsOpenedByBenchmark.forEach(txnId -> { + txnInfos.forEach(txnInfo -> { + if (txnInfo.getId() == txnId) { + notCleanedTxns.add(txnId); + } + }); + }); + + if (notCleanedTxns.size() != 0) { + return false; + } else return true; + }); + + return false; + } +} diff --git standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSBenchmarks.java standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSBenchmarks.java index d80c290b60..7c6f3431b0 100644 --- standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSBenchmarks.java +++ standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSBenchmarks.java @@ -19,8 +19,6 @@ package org.apache.hadoop.hive.metastore.tools; import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; -import org.apache.hadoop.hive.metastore.TableType; -import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.thrift.TException; @@ -28,22 +26,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.URI; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.stream.IntStream; import static org.apache.hadoop.hive.metastore.tools.Util.addManyPartitions; import static org.apache.hadoop.hive.metastore.tools.Util.addManyPartitionsNoException; -import static org.apache.hadoop.hive.metastore.tools.Util.createSchema; -import static org.apache.hadoop.hive.metastore.tools.Util.generatePartitionNames; -import static java.util.concurrent.Executors.newFixedThreadPool; import static org.apache.hadoop.hive.metastore.tools.Util.throwingSupplierWrapper; /** @@ -113,7 +103,7 @@ static DescriptiveStatistics benchmarkDeleteWithPartitions(@NotNull MicroBenchma return bench.measure( () -> throwingSupplierWrapper(() -> { - createPartitionedTable(client, dbName, tableName); + BenchmarkUtils.createPartitionedTable(client, dbName, tableName); addManyPartitions(client, dbName, tableName, parameters, Collections.singletonList("d"), howMany); return true; @@ -128,7 +118,7 @@ static DescriptiveStatistics benchmarkGetTable(@NotNull MicroBenchmark bench, String dbName = data.dbName; String tableName = data.tableName; - createPartitionedTable(client, dbName, tableName); + BenchmarkUtils.createPartitionedTable(client, dbName, tableName); try { return bench.measure(() -> throwingSupplierWrapper(() -> client.getTable(dbName, tableName))); @@ -146,11 +136,11 @@ static DescriptiveStatistics benchmarkListTables(@NotNull MicroBenchmark bench, // Create a bunch of tables String format = "tmp_table_%d"; try { - createManyTables(client, count, dbName, format); + BenchmarkUtils.createManyTables(client, count, dbName, format); return bench.measure(() -> throwingSupplierWrapper(() -> client.getAllTables(dbName, null))); } finally { - dropManyTables(client, count, dbName, format); + BenchmarkUtils.dropManyTables(client, count, dbName, format); } } @@ -160,7 +150,7 @@ static DescriptiveStatistics benchmarkCreatePartition(@NotNull MicroBenchmark be String dbName = data.dbName; String tableName = data.tableName; - createPartitionedTable(client, dbName, tableName); + BenchmarkUtils.createPartitionedTable(client, dbName, tableName); final List values = Collections.singletonList("d1"); try { Table t = client.getTable(dbName, tableName); @@ -185,7 +175,7 @@ static DescriptiveStatistics benchmarkListPartition(@NotNull MicroBenchmark benc String dbName = data.dbName; String tableName = data.tableName; - createPartitionedTable(client, dbName, tableName); + BenchmarkUtils.createPartitionedTable(client, dbName, tableName); try { addManyPartitions(client, dbName, tableName, null, Collections.singletonList("d"), 1); @@ -207,7 +197,7 @@ static DescriptiveStatistics benchmarkListManyPartitions(@NotNull MicroBenchmark String dbName = data.dbName; String tableName = data.tableName; - createPartitionedTable(client, dbName, tableName); + BenchmarkUtils.createPartitionedTable(client, dbName, tableName); try { addManyPartitions(client, dbName, tableName, null, Collections.singletonList("d"), howMany); LOG.debug("Created {} partitions", howMany); @@ -229,7 +219,7 @@ static DescriptiveStatistics benchmarkGetPartitions(@NotNull MicroBenchmark benc String dbName = data.dbName; String tableName = data.tableName; - createPartitionedTable(client, dbName, tableName); + BenchmarkUtils.createPartitionedTable(client, dbName, tableName); try { addManyPartitions(client, dbName, tableName, null, Collections.singletonList("d"), howMany); LOG.debug("Created {} partitions", howMany); @@ -250,7 +240,7 @@ static DescriptiveStatistics benchmarkDropPartition(@NotNull MicroBenchmark benc String dbName = data.dbName; String tableName = data.tableName; - createPartitionedTable(client, dbName, tableName); + BenchmarkUtils.createPartitionedTable(client, dbName, tableName); final List values = Collections.singletonList("d1"); try { Table t = client.getTable(dbName, tableName); @@ -277,7 +267,7 @@ static DescriptiveStatistics benchmarkCreatePartitions(@NotNull MicroBenchmark b String dbName = data.dbName; String tableName = data.tableName; - createPartitionedTable(client, dbName, tableName); + BenchmarkUtils.createPartitionedTable(client, dbName, tableName); try { return bench.measure( null, @@ -298,7 +288,7 @@ static DescriptiveStatistics benchmarkDropPartitions(@NotNull MicroBenchmark ben String dbName = data.dbName; String tableName = data.tableName; - createPartitionedTable(client, dbName, tableName); + BenchmarkUtils.createPartitionedTable(client, dbName, tableName); try { return bench.measure( () -> addManyPartitionsNoException(client, dbName, tableName, null, @@ -319,7 +309,7 @@ static DescriptiveStatistics benchmarkGetPartitionNames(@NotNull MicroBenchmark String dbName = data.dbName; String tableName = data.tableName; - createPartitionedTable(client, dbName, tableName); + BenchmarkUtils.createPartitionedTable(client, dbName, tableName); try { addManyPartitionsNoException(client, dbName, tableName, null, Collections.singletonList("d"), count); @@ -338,7 +328,7 @@ static DescriptiveStatistics benchmarkGetPartitionsByName(@NotNull MicroBenchmar String dbName = data.dbName; String tableName = data.tableName; - createPartitionedTable(client, dbName, tableName); + BenchmarkUtils.createPartitionedTable(client, dbName, tableName); try { addManyPartitionsNoException(client, dbName, tableName, null, Collections.singletonList("d"), count); @@ -361,7 +351,7 @@ static DescriptiveStatistics benchmarkRenameTable(@NotNull MicroBenchmark bench, String dbName = data.dbName; String tableName = data.tableName; - createPartitionedTable(client, dbName, tableName); + BenchmarkUtils.createPartitionedTable(client, dbName, tableName); try { addManyPartitionsNoException(client, dbName, tableName, null, Collections.singletonList("d"), count); @@ -398,7 +388,7 @@ static DescriptiveStatistics benchmarkDropDatabase(@NotNull MicroBenchmark bench return bench.measure( () -> { throwingSupplierWrapper(() -> client.createDatabase(dbName)); - createManyTables(client, count, dbName, "tmp_table_%d"); + BenchmarkUtils.createManyTables(client, count, dbName, "tmp_table_%d"); }, () -> throwingSupplierWrapper(() -> client.dropDatabase(dbName)), null @@ -417,33 +407,37 @@ static DescriptiveStatistics benchmarkOpenTxns(@NotNull MicroBenchmark bench, () -> throwingSupplierWrapper(() -> client.abortTxns(client.getOpenTxns()))); } - private static void createManyTables(HMSClient client, int howMany, String dbName, String format) { - List columns = createSchema(new ArrayList<>(Arrays.asList("name", "string"))); - List partitions = createSchema(new ArrayList<>(Arrays.asList("date", "string"))); - IntStream.range(0, howMany) - .forEach(i -> - throwingSupplierWrapper(() -> client.createTable( - new Util.TableBuilder(dbName, String.format(format, i)) - .withType(TableType.MANAGED_TABLE) - .withColumns(columns) - .withPartitionKeys(partitions) - .build()))); - } + static DescriptiveStatistics benchmarkAllocateTableWriteIds(@NotNull MicroBenchmark bench, + @NotNull BenchData data, + int howMany) { + final HMSClient client = data.getClient(); - private static void dropManyTables(HMSClient client, int howMany, String dbName, String format) { - IntStream.range(0, howMany) - .forEach(i -> - throwingSupplierWrapper(() -> client.dropTable(dbName, String.format(format, i)))); + return bench.measure( + () -> throwingSupplierWrapper(() -> client.openTxn(howMany)), + () -> throwingSupplierWrapper(() -> client.allocateTableWriteIds("test_db", "test_tbl", client.getOpenTxns())), + () -> throwingSupplierWrapper(() -> client.abortTxns(client.getOpenTxns())) + ); } - // Create a simple table with a single column and single partition - private static void createPartitionedTable(HMSClient client, String dbName, String tableName) { - throwingSupplierWrapper(() -> client.createTable( - new Util.TableBuilder(dbName, tableName) - .withType(TableType.MANAGED_TABLE) - .withColumns(createSchema(Collections.singletonList("name:string"))) - .withPartitionKeys(createSchema(Collections.singletonList("date"))) - .build())); + static DescriptiveStatistics benchmarkGetValidWriteIds(@NotNull MicroBenchmark bench, + @NotNull BenchData data, + int howMany) { + final HMSClient client = data.getClient(); + String dbName = data.dbName; + List tableNames = new ArrayList<>(); + + return bench.measure( + () -> { + BenchmarkUtils.createManyTables(client, howMany, dbName, "tmp_table_%d"); + for (int i = 0; i < howMany; i++) { + tableNames.add(dbName + ".tmp_table_" + i); + } + }, + () -> throwingSupplierWrapper(() -> client.getValidWriteIds(tableNames)), + () -> { + BenchmarkUtils.dropManyTables(client, howMany, dbName, "tmp_table_%d"); + } + ); } static DescriptiveStatistics benchmarkGetNotificationId(@NotNull MicroBenchmark benchmark, @@ -453,5 +447,4 @@ static DescriptiveStatistics benchmarkGetNotificationId(@NotNull MicroBenchmark throwingSupplierWrapper(client::getCurrentNotificationId)); } - } diff --git standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSConfig.java standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSConfig.java new file mode 100644 index 0000000000..86b666cef4 --- /dev/null +++ standalone-metastore/metastore-tools/metastore-benchmarks/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSConfig.java @@ -0,0 +1,29 @@ +package org.apache.hadoop.hive.metastore.tools; + +import static org.apache.hadoop.hive.metastore.tools.Util.getServerUri; + +public class HMSConfig { + + private static final HMSConfig instance = new HMSConfig(); + + private String host; + private Integer port; + private String confDir; + + private HMSConfig(){} + + public static HMSConfig getInstance(){ + return instance; + } + + public void init(String host, Integer port, String confDir){ + this.host = host; + this.port = port; + this.confDir = confDir; + } + + public HMSClient newClient() throws Exception { + return new HMSClient(getServerUri(host, String.valueOf(port)), confDir); + } + +} diff --git standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkSuite.java standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkSuite.java index 5211082a7d..d32e971eb2 100644 --- standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkSuite.java +++ standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkSuite.java @@ -75,7 +75,7 @@ // List of benchmarks. All benchmarks are executed in the order // they are inserted private final List benchmarks = new ArrayList<>(); - // Once benchmarks are executed, results are stored in TreeMap to prserve the order. + // Once benchmarks are executed, results are stored in TreeMap to preserve the order. private final Map result = new TreeMap<>(); // Whether sanitizing of results is requested private boolean doSanitize = false; @@ -115,7 +115,7 @@ public BenchmarkSuite doSanitize(boolean sanitize) { } /** - * Run all benchmarks in the 'names' list. + * Run all benchmarks in the 'names' list. * @param names list of benchmarks to run * @return this to allow chaining */ diff --git standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSClient.java standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSClient.java index 4e75edeae6..8913cb2414 100644 --- standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSClient.java +++ standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSClient.java @@ -20,19 +20,25 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.api.AbortTxnRequest; import org.apache.hadoop.hive.metastore.api.AbortTxnsRequest; +import org.apache.hadoop.hive.metastore.api.AllocateTableWriteIdsRequest; import org.apache.hadoop.hive.metastore.api.CommitTxnRequest; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.DropPartitionsRequest; import org.apache.hadoop.hive.metastore.api.DropPartitionsResult; import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse; +import org.apache.hadoop.hive.metastore.api.GetValidWriteIdsRequest; +import org.apache.hadoop.hive.metastore.api.LockRequest; +import org.apache.hadoop.hive.metastore.api.LockResponse; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.OpenTxnRequest; +import org.apache.hadoop.hive.metastore.api.OpenTxnsResponse; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.RequestPartsSpec; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore; +import org.apache.hadoop.hive.metastore.api.TxnInfo; +import org.apache.hadoop.hive.metastore.api.TxnType; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; @@ -53,9 +59,11 @@ import javax.security.auth.login.LoginException; import java.io.File; import java.io.IOException; +import java.net.InetAddress; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; +import java.net.UnknownHostException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Arrays; @@ -326,11 +334,6 @@ void appendPartition(@NotNull String dbName, @NotNull String tableName, client.append_partition_with_environment_context(dbName, tableName, partitionValues, null); } - boolean openTxn(int numTxns) throws TException { - client.open_txns(new OpenTxnRequest(numTxns, "Test", "Host")); - return true; - } - List getOpenTxns() throws TException { GetOpenTxnsResponse txns = client.get_open_txns(); List openTxns = new ArrayList<>(); @@ -345,21 +348,44 @@ boolean openTxn(int numTxns) throws TException { return openTxns; } + List getOpenTxnsInfo() throws TException { + return client.get_open_txns_info().getOpen_txns(); + } + boolean commitTxn(long txnId) throws TException { client.commit_txn(new CommitTxnRequest(txnId)); return true; } - boolean abortTxn(long txnId) throws TException { - client.abort_txn(new AbortTxnRequest(txnId)); + boolean abortTxns(List txnIds) throws TException { + client.abort_txns(new AbortTxnsRequest(txnIds)); return true; } - boolean abortTxns(List txnIds) throws TException { - client.abort_txns(new AbortTxnsRequest(txnIds)); + boolean allocateTableWriteIds(String dbName, String tableName, List openTxns) throws TException { + AllocateTableWriteIdsRequest awiRqst = new AllocateTableWriteIdsRequest(dbName, tableName); + openTxns.forEach(t -> { + awiRqst.addToTxnIds(t); + }); + + client.allocate_table_write_ids(awiRqst); return true; } + boolean getValidWriteIds(List fullTableNames) throws TException { + client.get_valid_write_ids(new GetValidWriteIdsRequest(fullTableNames)); + return true; + } + + LockResponse lock(@NotNull LockRequest rqst) throws TException { + return client.lock(rqst); + } + + List openTxn(int howMany) throws TException { + OpenTxnsResponse txns = openTxnsIntr("", howMany, null); + return txns.getTxn_ids(); + } + private TTransport open(Configuration conf, @NotNull URI uri) throws TException, IOException, LoginException { boolean useSSL = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.USE_SSL); @@ -459,6 +485,21 @@ private TTransport open(Configuration conf, @NotNull URI uri) throws return transport; } + private OpenTxnsResponse openTxnsIntr(String user, int numTxns, TxnType txnType) throws TException { + String hostname; + try { + hostname = InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + LOG.error("Unable to resolve my host name " + e.getMessage()); + throw new RuntimeException(e); + } + OpenTxnRequest rqst = new OpenTxnRequest(numTxns, user, hostname); + if (txnType != null) { + rqst.setTxn_type(txnType); + } + return client.open_txns(rqst); + } + @Override public void close() throws Exception { if (transport != null && transport.isOpen()) { diff --git standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/Util.java standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/Util.java index 101d6759c5..fd06db6276 100644 --- standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/Util.java +++ standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/Util.java @@ -21,8 +21,12 @@ import com.google.common.base.Joiner; import com.google.common.net.HostAndPort; import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.metastore.api.DataOperationType; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.LockComponent; +import org.apache.hadoop.hive.metastore.api.LockLevel; +import org.apache.hadoop.hive.metastore.api.LockType; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.SerDeInfo; @@ -232,6 +236,7 @@ public Database build() { private TableBuilder() { dbName = null; tableName = null; + parameters.putIfAbsent("transactional", "true"); } TableBuilder(String dbName, String tableName) { @@ -383,6 +388,99 @@ Partition build() { } } + public static class LockComponentBuilder { + private LockComponent component; + private boolean tableNameSet; + private boolean partNameSet; + + public LockComponentBuilder() { + component = new LockComponent(); + tableNameSet = partNameSet = false; + } + + /** + * Set the lock to be exclusive. + * @return reference to this builder + */ + public LockComponentBuilder setExclusive() { + component.setType(LockType.EXCLUSIVE); + return this; + } + + /** + * Set the lock to be semi-shared. + * @return reference to this builder + */ + public LockComponentBuilder setSemiShared() { + component.setType(LockType.SHARED_WRITE); + return this; + } + + /** + * Set the lock to be shared. + * @return reference to this builder + */ + public LockComponentBuilder setShared() { + component.setType(LockType.SHARED_READ); + return this; + } + + /** + * Set the database name. + * @param dbName database name + * @return reference to this builder + */ + public LockComponentBuilder setDbName(String dbName) { + component.setDbname(dbName); + return this; + } + + public LockComponentBuilder setIsTransactional(boolean t) { + component.setIsTransactional(t); + return this; + } + + public LockComponentBuilder setOperationType(DataOperationType dop) { + component.setOperationType(dop); + return this; + } + + /** + * Set the table name. + * @param tableName table name + * @return reference to this builder + */ + public LockComponentBuilder setTableName(String tableName) { + component.setTablename(tableName); + tableNameSet = true; + return this; + } + + /** + * Set the partition name. + * @param partitionName partition name + * @return reference to this builder + */ + public LockComponentBuilder setPartitionName(String partitionName) { + component.setPartitionname(partitionName); + partNameSet = true; + return this; + } + + public LockComponent build() { + LockLevel level = LockLevel.DB; + if (tableNameSet) level = LockLevel.TABLE; + if (partNameSet) level = LockLevel.PARTITION; + component.setLevel(level); + return component; + } + + public LockComponent setLock(LockType type) { + component.setType(type); + return component; + } + } + /** * Create table schema from parameters. *