diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool.java new file mode 100644 index 0000000..86db2cd --- /dev/null +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool.java @@ -0,0 +1,553 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hive.metastore.hbase; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.Function; +import org.apache.hadoop.hive.metastore.api.FunctionType; +import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege; +import org.apache.hadoop.hive.metastore.api.HiveObjectRef; +import org.apache.hadoop.hive.metastore.api.HiveObjectType; +import org.apache.hadoop.hive.metastore.api.LongColumnStatsData; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.PrincipalType; +import org.apache.hadoop.hive.metastore.api.PrivilegeBag; +import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo; +import org.apache.hadoop.hive.metastore.api.Role; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.api.StringColumnStatsData; +import org.apache.hadoop.hive.metastore.api.Table; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class TestHBaseSchemaTool extends HBaseIntegrationTests { + private static final Log LOG = LogFactory.getLog(TestHBaseSchemaTool.class.getName()); + private String lsep = System.getProperty("line.separator"); + + @BeforeClass + public static void startup() throws Exception { + HBaseIntegrationTests.startMiniCluster(); + } + + @AfterClass + public static void shutdown() throws Exception { + HBaseIntegrationTests.shutdownMiniCluster(); + } + + @Before + public void setup() throws IOException { + setupHBaseStore(); + } + + @Test + public void listTables() throws Exception { + ByteArrayOutputStream outStr = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(outStr); + ByteArrayOutputStream errStr = new ByteArrayOutputStream(); + PrintStream err = new PrintStream(errStr); + + new HBaseSchemaTool().go(true, null, null, null, conf, out, err); + Assert.assertEquals(StringUtils.join(HBaseReadWrite.tableNames, lsep) + lsep, + outStr.toString()); + } + + @Test + public void bogusTable() throws Exception { + ByteArrayOutputStream outStr = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(outStr); + ByteArrayOutputStream errStr = new ByteArrayOutputStream(); + PrintStream err = new PrintStream(errStr); + + new HBaseSchemaTool().go(false, "nosuch", null, null, conf, out, err); + Assert.assertEquals("Unknown table: nosuch" + lsep, errStr.toString()); + } + + @Test + public void noSuchDb() throws Exception { + ByteArrayOutputStream outStr = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(outStr); + ByteArrayOutputStream errStr = new ByteArrayOutputStream(); + PrintStream err = new PrintStream(errStr); + + new HBaseSchemaTool().go(false, HBaseReadWrite.DB_TABLE, "nosuch", null, conf, out, err); + Assert.assertEquals("No such database: nosuch" + lsep, outStr.toString()); + } + + @Test + public void noMatchingDb() throws Exception { + ByteArrayOutputStream outStr = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(outStr); + ByteArrayOutputStream errStr = new ByteArrayOutputStream(); + PrintStream err = new PrintStream(errStr); + + new HBaseSchemaTool().go(false, HBaseReadWrite.DB_TABLE, null, "nomatch", conf, out, err); + Assert.assertEquals("No matching database: nomatch" + lsep, outStr.toString()); + } + + @Test + public void noSuchRole() throws Exception { + ByteArrayOutputStream outStr = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(outStr); + ByteArrayOutputStream errStr = new ByteArrayOutputStream(); + PrintStream err = new PrintStream(errStr); + + new HBaseSchemaTool().go(false, HBaseReadWrite.ROLE_TABLE, "nosuch", null, conf, out, err); + Assert.assertEquals("No such role: nosuch" + lsep, outStr.toString()); + } + + @Test + public void noMatchingRole() throws Exception { + ByteArrayOutputStream outStr = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(outStr); + ByteArrayOutputStream errStr = new ByteArrayOutputStream(); + PrintStream err = new PrintStream(errStr); + + new HBaseSchemaTool().go(false, HBaseReadWrite.ROLE_TABLE, null, "nomatch", conf, out, err); + Assert.assertEquals("No matching role: nomatch" + lsep, outStr.toString()); + } + + @Test + public void noSuchUser() throws Exception { + ByteArrayOutputStream outStr = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(outStr); + ByteArrayOutputStream errStr = new ByteArrayOutputStream(); + PrintStream err = new PrintStream(errStr); + + new HBaseSchemaTool().go(false, HBaseReadWrite.USER_TO_ROLE_TABLE, "nosuch", null, conf, out, err); + Assert.assertEquals("No such user: nosuch" + lsep, outStr.toString()); + } + + @Test + public void noMatchingUser() throws Exception { + ByteArrayOutputStream outStr = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(outStr); + ByteArrayOutputStream errStr = new ByteArrayOutputStream(); + PrintStream err = new PrintStream(errStr); + + new HBaseSchemaTool().go(false, HBaseReadWrite.USER_TO_ROLE_TABLE, null, "nomatch", conf, out, err); + Assert.assertEquals("No matching user: nomatch" + lsep, outStr.toString()); + } + + @Test + public void noSuchFunction() throws Exception { + ByteArrayOutputStream outStr = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(outStr); + ByteArrayOutputStream errStr = new ByteArrayOutputStream(); + PrintStream err = new PrintStream(errStr); + + new HBaseSchemaTool().go(false, HBaseReadWrite.FUNC_TABLE, "nosuch", null, conf, out, err); + Assert.assertEquals("No such function: nosuch" + lsep, outStr.toString()); + } + + @Test + public void noMatchingFunction() throws Exception { + ByteArrayOutputStream outStr = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(outStr); + ByteArrayOutputStream errStr = new ByteArrayOutputStream(); + PrintStream err = new PrintStream(errStr); + + new HBaseSchemaTool().go(false, HBaseReadWrite.FUNC_TABLE, null, "nomatch", conf, out, + err); + Assert.assertEquals("No matching function: nomatch" + lsep, outStr.toString()); + } + + @Test + public void noSuchTable() throws Exception { + ByteArrayOutputStream outStr = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(outStr); + ByteArrayOutputStream errStr = new ByteArrayOutputStream(); + PrintStream err = new PrintStream(errStr); + + new HBaseSchemaTool().go(false, HBaseReadWrite.TABLE_TABLE, "nosuch", null, conf, out, err); + Assert.assertEquals("No such table: nosuch" + lsep, outStr.toString()); + } + + @Test + public void noMatchingTable() throws Exception { + ByteArrayOutputStream outStr = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(outStr); + ByteArrayOutputStream errStr = new ByteArrayOutputStream(); + PrintStream err = new PrintStream(errStr); + + new HBaseSchemaTool().go(false, HBaseReadWrite.TABLE_TABLE, null, "nomatch", conf, out, err); + Assert.assertEquals("No matching table: nomatch" + lsep, outStr.toString()); + } + + @Test + public void noSuchPart() throws Exception { + ByteArrayOutputStream outStr = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(outStr); + ByteArrayOutputStream errStr = new ByteArrayOutputStream(); + PrintStream err = new PrintStream(errStr); + + new HBaseSchemaTool().go(false, HBaseReadWrite.PART_TABLE, "nosuch", null, conf, out, err); + Assert.assertEquals("No such partition: nosuch" + lsep, outStr.toString()); + } + + @Test + public void noMatchingPart() throws Exception { + ByteArrayOutputStream outStr = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(outStr); + ByteArrayOutputStream errStr = new ByteArrayOutputStream(); + PrintStream err = new PrintStream(errStr); + + new HBaseSchemaTool().go(false, HBaseReadWrite.PART_TABLE, null, "nomatch", conf, out, err); + Assert.assertEquals("No matching partition: nomatch" + lsep, outStr.toString()); + } + + @Test + public void noSuchStorageDescriptor() throws Exception { + ByteArrayOutputStream outStr = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(outStr); + ByteArrayOutputStream errStr = new ByteArrayOutputStream(); + PrintStream err = new PrintStream(errStr); + + // Strangly enough things don't come back quite the same when going through the Base64 + // encode/decode. + new HBaseSchemaTool().go(false, HBaseReadWrite.SD_TABLE, "nosuch", null, conf, out, err); + Assert.assertEquals("No such storage descriptor: nosucg" + lsep, outStr.toString()); + } + + @Test + public void oneMondoTest() throws Exception { + // This is a pain to do in one big test, but we have to control the order so that we have tests + // without dbs, etc. + HBaseSchemaTool tool = new HBaseSchemaTool(); + + ByteArrayOutputStream outStr = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(outStr); + ByteArrayOutputStream errStr = new ByteArrayOutputStream(); + PrintStream err = new PrintStream(errStr); + + // This needs to be up front before we create any tables or partitions + tool.go(false, HBaseReadWrite.SD_TABLE, null, "whatever", conf, out, err); + Assert.assertEquals("No storage descriptors" + lsep, outStr.toString()); + + // This one needs to be up front too + outStr = new ByteArrayOutputStream(); + out = new PrintStream(outStr); + tool.go(false, HBaseReadWrite.SEQUENCES_TABLE, null, "whatever", conf, out, err); + Assert.assertEquals("No sequences" + lsep, outStr.toString()); + + // Create some databases + String[] dbNames = new String[3]; + for (int i = 0; i < dbNames.length; i++) { + dbNames[i] = "db" + i; + Database db = new Database(dbNames[i], "no description", "file:///tmp", emptyParameters); + store.createDatabase(db); + } + + outStr = new ByteArrayOutputStream(); + out = new PrintStream(outStr); + tool.go(false, HBaseReadWrite.DB_TABLE, "db0", null, conf, out, err); + Assert.assertEquals("{\"name\":\"db0\",\"description\":\"no description\"," + + "\"locationUri\":\"file:///tmp\",\"parameters\":{}}" + lsep, outStr.toString()); + + outStr = new ByteArrayOutputStream(); + out = new PrintStream(outStr); + tool.go(false, HBaseReadWrite.DB_TABLE, null, ".*", conf, out, err); + Assert.assertEquals("{\"name\":\"db0\",\"description\":\"no description\"," + + "\"locationUri\":\"file:///tmp\",\"parameters\":{}}" + lsep + + "{\"name\":\"db1\",\"description\":\"no description\"," + + "\"locationUri\":\"file:///tmp\",\"parameters\":{}}" + lsep + + "{\"name\":\"db2\",\"description\":\"no description\"," + + "\"locationUri\":\"file:///tmp\",\"parameters\":{}}" + lsep, + outStr.toString()); + + outStr = new ByteArrayOutputStream(); + out = new PrintStream(outStr); + tool.go(false, HBaseReadWrite.DB_TABLE, null, "db[12]", conf, out, err); + Assert.assertEquals("{\"name\":\"db1\",\"description\":\"no description\"," + + "\"locationUri\":\"file:///tmp\",\"parameters\":{}}" + lsep + + "{\"name\":\"db2\",\"description\":\"no description\"," + + "\"locationUri\":\"file:///tmp\",\"parameters\":{}}" + lsep, + outStr.toString()); + + String[] roleNames = new String[2]; + for (int i = 0; i < roleNames.length; i++) { + roleNames[i] = "role" + i; + store.addRole(roleNames[i], "me"); + } + outStr = new ByteArrayOutputStream(); + out = new PrintStream(outStr); + tool.go(false, HBaseReadWrite.ROLE_TABLE, null, "role.", conf, out, err); + Assert.assertEquals("{\"roleName\":\"role0\",\"createTime\":now,\"ownerName\":\"me\"}" + + lsep + "{\"roleName\":\"role1\",\"createTime\":now,\"ownerName\":\"me\"}" + lsep, + outStr.toString().replaceAll("createTime\":[0-9]+", "createTime\":now")); + + outStr = new ByteArrayOutputStream(); + out = new PrintStream(outStr); + tool.go(false, HBaseReadWrite.ROLE_TABLE, "role1", null, conf, out, err); + Assert.assertEquals("{\"roleName\":\"role1\",\"createTime\":now,\"ownerName\":\"me\"}" + lsep, + outStr.toString().replaceAll("createTime\":[0-9]+", "createTime\":now")); + + Role role1 = store.getRole("role1"); + store.grantRole(role1, "fred", PrincipalType.USER, "me", PrincipalType.USER, false); + store.grantRole(role1, "joanne", PrincipalType.USER, "me", PrincipalType.USER, false); + + outStr = new ByteArrayOutputStream(); + out = new PrintStream(outStr); + tool.go(false, HBaseReadWrite.USER_TO_ROLE_TABLE, null, ".*", conf, out, err); + Assert.assertEquals("fred: role1" + lsep + "joanne: role1" + lsep, outStr.toString()); + + outStr = new ByteArrayOutputStream(); + out = new PrintStream(outStr); + tool.go(false, HBaseReadWrite.USER_TO_ROLE_TABLE, "joanne", null, conf, out, err); + Assert.assertEquals("role1" + lsep, outStr.toString()); + + String[] funcNames = new String[3]; + for (int i = 0; i < funcNames.length; i++) { + funcNames[i] = "func" + i; + Function function = new Function(funcNames[i], "db1", "Function", "me", PrincipalType.USER, 0, + FunctionType.JAVA, null); + store.createFunction(function); + } + outStr = new ByteArrayOutputStream(); + out = new PrintStream(outStr); + tool.go(false, HBaseReadWrite.FUNC_TABLE, "db1.func0", null, conf, out, err); + Assert.assertEquals("{\"functionName\":\"func0\",\"dbName\":\"db1\"," + + "\"className\":\"Function\",\"ownerName\":\"me\",\"ownerType\":1,\"createTime\":0," + + "\"functionType\":1}" + lsep, outStr.toString()); + + outStr = new ByteArrayOutputStream(); + out = new PrintStream(outStr); + tool.go(false, HBaseReadWrite.FUNC_TABLE, null, ".*", conf, out, err); + Assert.assertEquals("{\"functionName\":\"func0\",\"dbName\":\"db1\"," + + "\"className\":\"Function\",\"ownerName\":\"me\",\"ownerType\":1,\"createTime\":0," + + "\"functionType\":1}" + lsep + + "{\"functionName\":\"func1\",\"dbName\":\"db1\"," + + "\"className\":\"Function\",\"ownerName\":\"me\",\"ownerType\":1,\"createTime\":0," + + "\"functionType\":1}" + lsep + + "{\"functionName\":\"func2\",\"dbName\":\"db1\"," + + "\"className\":\"Function\",\"ownerName\":\"me\",\"ownerType\":1,\"createTime\":0," + + "\"functionType\":1}" + lsep, outStr.toString()); + + outStr = new ByteArrayOutputStream(); + out = new PrintStream(outStr); + tool.go(false, HBaseReadWrite.FUNC_TABLE, null, "db1.func[12]", conf, out, err); + Assert.assertEquals("{\"functionName\":\"func1\",\"dbName\":\"db1\"," + + "\"className\":\"Function\",\"ownerName\":\"me\",\"ownerType\":1,\"createTime\":0," + + "\"functionType\":1}" + lsep + + "{\"functionName\":\"func2\",\"dbName\":\"db1\"," + + "\"className\":\"Function\",\"ownerName\":\"me\",\"ownerType\":1,\"createTime\":0," + + "\"functionType\":1}" + lsep, outStr.toString()); + + + outStr = new ByteArrayOutputStream(); + out = new PrintStream(outStr); + tool.go(false, HBaseReadWrite.GLOBAL_PRIVS_TABLE, null, null, conf, out, err); + Assert.assertEquals("No global privileges" + lsep, outStr.toString()); + + List privileges = new ArrayList<>(); + HiveObjectRef hiveObjRef = new HiveObjectRef(HiveObjectType.GLOBAL, "db0", "tab0", null, + null); + PrivilegeGrantInfo grantInfo = + new PrivilegeGrantInfo("read", 0, "me", PrincipalType.USER, false); + HiveObjectPrivilege hop = new HiveObjectPrivilege(hiveObjRef, "user", PrincipalType.USER, + grantInfo); + privileges.add(hop); + + grantInfo = new PrivilegeGrantInfo("create", 0, "me", PrincipalType.USER, true); + hop = new HiveObjectPrivilege(hiveObjRef, "user", PrincipalType.USER, grantInfo); + privileges.add(hop); + + PrivilegeBag pBag = new PrivilegeBag(privileges); + store.grantPrivileges(pBag); + + outStr = new ByteArrayOutputStream(); + out = new PrintStream(outStr); + tool.go(false, HBaseReadWrite.GLOBAL_PRIVS_TABLE, null, null, conf, out, err); + Assert.assertEquals( + "{\"userPrivileges\":{\"user\":[{\"privilege\":\"read\",\"createTime\":0," + + "\"grantor\":\"me\",\"grantorType\":1,\"grantOption\":0},{\"privilege\":\"create\"," + + "\"createTime\":0,\"grantor\":\"me\",\"grantorType\":1,\"grantOption\":1}]}," + + "\"rolePrivileges\":{}}" + lsep, outStr.toString()); + + + String[] tableNames = new String[3]; + for (int i = 0; i < tableNames.length; i++) { + tableNames[i] = "tab" + i; + StorageDescriptor sd = new StorageDescriptor(Arrays.asList(new FieldSchema("col1", "int", + ""), new FieldSchema("col2", "varchar(32)", "")), + "/tmp", null, null, false, 0, null, null, null, Collections.emptyMap()); + Table tab = new Table(tableNames[i], dbNames[0], "me", 0, 0, 0, sd, + Arrays.asList(new FieldSchema("pcol1", "string", ""), + new FieldSchema("pcol2", "string", "")), + Collections.emptyMap(), null, null, null); + store.createTable(tab); + } + + ColumnStatisticsDesc tableStatsDesc = new ColumnStatisticsDesc(false, "db0", "tab0"); + ColumnStatisticsData tcsd = new ColumnStatisticsData(); + LongColumnStatsData tlcsd = new LongColumnStatsData(1, 2); + tlcsd.setLowValue(-95); + tlcsd.setHighValue(95); + tcsd.setLongStats(tlcsd); + ColumnStatisticsData tcsd2 = new ColumnStatisticsData(); + tcsd2.setStringStats(new StringColumnStatsData(97, 18.78, 29, 397)); + List tcsos = Arrays.asList( + new ColumnStatisticsObj("col1", "int", tcsd), + new ColumnStatisticsObj("col2", "varchar(32)", tcsd2)); + ColumnStatistics tStatObj = new ColumnStatistics(tableStatsDesc, tcsos); + store.updateTableColumnStatistics(tStatObj); + + outStr = new ByteArrayOutputStream(); + out = new PrintStream(outStr); + tool.go(false, HBaseReadWrite.TABLE_TABLE, "db0.tab1", null, conf, out, err); + Assert.assertEquals("{\"tableName\":\"tab1\",\"dbName\":\"db0\",\"owner\":\"me\"," + + "\"createTime\":0,\"lastAccessTime\":0,\"retention\":0," + + "\"partitionKeys\":[{\"name\":\"pcol1\",\"type\":\"string\",\"comment\":\"\"}," + + "{\"name\":\"pcol2\",\"type\":\"string\",\"comment\":\"\"}],\"parameters\":{}," + + "\"tableType\":\"\",\"temporary\":0} sdHash: qQTgZAi5VzgpozzFGmIVTQ stats:" + lsep, + outStr.toString()); + + outStr = new ByteArrayOutputStream(); + out = new PrintStream(outStr); + tool.go(false, HBaseReadWrite.TABLE_TABLE, null, "db0.*", conf, out, err); + Assert.assertEquals("{\"tableName\":\"tab0\",\"dbName\":\"db0\",\"owner\":\"me\"," + + "\"createTime\":0,\"lastAccessTime\":0,\"retention\":0," + + "\"partitionKeys\":[{\"name\":\"pcol1\",\"type\":\"string\",\"comment\":\"\"}," + + "{\"name\":\"pcol2\",\"type\":\"string\",\"comment\":\"\"}],\"parameters\":{}," + + "\"tableType\":\"\",\"temporary\":0} sdHash: qQTgZAi5VzgpozzFGmIVTQ stats: column " + + "col1: {\"colName\":\"col1\",\"colType\":\"int\"," + + "\"statsData\":{\"longStats\":{\"lowValue\":-95,\"highValue\":95,\"numNulls\":1," + + "\"numDVs\":2}}} column col2: {\"colName\":\"col2\",\"colType\":\"varchar(32)\"," + + "\"statsData\":{\"stringStats\":{\"maxColLen\":97,\"avgColLen\":18.78," + + "\"numNulls\":29,\"numDVs\":397}}}" + lsep + + "{\"tableName\":\"tab1\",\"dbName\":\"db0\",\"owner\":\"me\",\"createTime\":0," + + "\"lastAccessTime\":0,\"retention\":0,\"partitionKeys\":[{\"name\":\"pcol1\"," + + "\"type\":\"string\",\"comment\":\"\"},{\"name\":\"pcol2\",\"type\":\"string\"," + + "\"comment\":\"\"}],\"parameters\":{},\"tableType\":\"\",\"temporary\":0} sdHash: " + + "qQTgZAi5VzgpozzFGmIVTQ stats:" + lsep + + "{\"tableName\":\"tab2\",\"dbName\":\"db0\",\"owner\":\"me\",\"createTime\":0," + + "\"lastAccessTime\":0,\"retention\":0,\"partitionKeys\":[{\"name\":\"pcol1\"," + + "\"type\":\"string\",\"comment\":\"\"},{\"name\":\"pcol2\",\"type\":\"string\"," + + "\"comment\":\"\"}],\"parameters\":{},\"tableType\":\"\",\"temporary\":0} sdHash: " + + "qQTgZAi5VzgpozzFGmIVTQ stats:" + lsep, outStr.toString()); + + List> partVals = Arrays.asList(Arrays.asList("a", "b"), Arrays.asList("c", "d")); + for (List pv : partVals) { + StorageDescriptor sd = new StorageDescriptor(Arrays.asList(new FieldSchema("col1", "int", + ""), new FieldSchema("col2", "varchar(32)", "")), + "/tmp", null, null, false, 0, null, null, null, Collections.emptyMap()); + Partition p = new Partition(pv, "db1", "tab1", 0, 0, sd, Collections.emptyMap()); + store.addPartition(p); + } + outStr = new ByteArrayOutputStream(); + out = new PrintStream(outStr); + tool.go(false, HBaseReadWrite.PART_TABLE, "db1.tab1.a.b", null, conf, out, err); + Assert.assertEquals("{\"values\":[\"a\",\"b\"],\"dbName\":\"db1\",\"tableName\":\"tab1\"," + + "\"createTime\":0,\"lastAccessTime\":0,\"parameters\":{}} sdHash: " + + "qQTgZAi5VzgpozzFGmIVTQ stats:" + lsep, outStr.toString()); + + ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(false, "db1", "tab1"); + statsDesc.setPartName("pcol1=c/pcol2=d"); + ColumnStatisticsData csd1 = new ColumnStatisticsData(); + LongColumnStatsData lcsd = new LongColumnStatsData(1, 2); + lcsd.setLowValue(-95); + lcsd.setHighValue(95); + csd1.setLongStats(lcsd); + ColumnStatisticsData csd2 = new ColumnStatisticsData(); + csd2.setStringStats(new StringColumnStatsData(97, 18.78, 29, 397)); + List csos = Arrays.asList( + new ColumnStatisticsObj("col1", "int", csd1), + new ColumnStatisticsObj("col2", "varchar(32)", csd2)); + ColumnStatistics statsObj = new ColumnStatistics(statsDesc, csos); + store.updatePartitionColumnStatistics(statsObj, partVals.get(1)); + + outStr = new ByteArrayOutputStream(); + out = new PrintStream(outStr); + tool.go(false, HBaseReadWrite.PART_TABLE, "db1.tab1.c.d", null, conf, out, err); + Assert.assertEquals("{\"values\":[\"c\",\"d\"],\"dbName\":\"db1\",\"tableName\":\"tab1\"," + + "\"createTime\":0,\"lastAccessTime\":0,\"parameters\":{}} sdHash: qQTgZAi5VzgpozzFGmIVTQ " + + "stats: column col1: {\"colName\":\"col1\",\"colType\":\"int\"," + + "\"statsData\":{\"longStats\":{\"lowValue\":-95,\"highValue\":95,\"numNulls\":1," + + "\"numDVs\":2}}} column col2: {\"colName\":\"col2\",\"colType\":\"varchar(32)\"," + + "\"statsData\":{\"stringStats\":{\"maxColLen\":97,\"avgColLen\":18.78,\"numNulls\":29," + + "\"numDVs\":397}}}" + lsep, outStr.toString()); + + outStr = new ByteArrayOutputStream(); + out = new PrintStream(outStr); + tool.go(false, HBaseReadWrite.PART_TABLE, null, "db1.tab1..*", conf, out, err); + Assert.assertEquals("{\"values\":[\"a\",\"b\"],\"dbName\":\"db1\",\"tableName\":\"tab1\"," + + "\"createTime\":0,\"lastAccessTime\":0,\"parameters\":{}} sdHash: qQTgZAi5VzgpozzFGmIVTQ " + + "stats:" + lsep + + "{\"values\":[\"c\",\"d\"],\"dbName\":\"db1\",\"tableName\":\"tab1\",\"createTime\":0," + + "\"lastAccessTime\":0,\"parameters\":{}} sdHash: qQTgZAi5VzgpozzFGmIVTQ stats: column " + + "col1: {\"colName\":\"col1\",\"colType\":\"int\"," + + "\"statsData\":{\"longStats\":{\"lowValue\":-95,\"highValue\":95,\"numNulls\":1," + + "\"numDVs\":2}}} column col2: {\"colName\":\"col2\",\"colType\":\"varchar(32)\"," + + "\"statsData\":{\"stringStats\":{\"maxColLen\":97,\"avgColLen\":18.78,\"numNulls\":29," + + "\"numDVs\":397}}}" + lsep, outStr.toString()); + + outStr = new ByteArrayOutputStream(); + out = new PrintStream(outStr); + tool.go(false, HBaseReadWrite.SD_TABLE, "qQTgZAi5VzgpozzFGmIVTQ", null, conf, out, err); + Assert.assertEquals("{\"cols\":[{\"name\":\"col1\",\"type\":\"int\",\"comment\":\"\"}," + + "{\"name\":\"col2\",\"type\":\"varchar(32)\",\"comment\":\"\"}],\"compressed\":0," + + "\"numBuckets\":0,\"bucketCols\":[],\"sortCols\":[],\"storedAsSubDirectories\":0}" + lsep, + outStr.toString()); + + outStr = new ByteArrayOutputStream(); + out = new PrintStream(outStr); + tool.go(false, HBaseReadWrite.SD_TABLE, null, "whatever", conf, out, err); + Assert.assertEquals("qQTgZAi5VzgpozzFGmIVTQ: {\"cols\":[{\"name\":\"col1\",\"type\":\"int\"," + + "\"comment\":\"\"}," + + "{\"name\":\"col2\",\"type\":\"varchar(32)\",\"comment\":\"\"}],\"compressed\":0," + + "\"numBuckets\":0,\"bucketCols\":[],\"sortCols\":[],\"storedAsSubDirectories\":0}" + lsep, + outStr.toString()); + + outStr = new ByteArrayOutputStream(); + out = new PrintStream(outStr); + tool.go(false, HBaseReadWrite.SECURITY_TABLE, null, "whatever", conf, out, err); + Assert.assertEquals("No security related entries" + lsep, outStr.toString()); + + store.addMasterKey("this be a key"); + store.addToken("tokenid", "delegation token"); + outStr = new ByteArrayOutputStream(); + out = new PrintStream(outStr); + tool.go(false, HBaseReadWrite.SECURITY_TABLE, null, "whatever", conf, out, err); + Assert.assertEquals("Master key 0: this be a key" + lsep + + "Delegation token tokenid: delegation token" + lsep, outStr.toString()); + + outStr = new ByteArrayOutputStream(); + out = new PrintStream(outStr); + tool.go(false, HBaseReadWrite.SEQUENCES_TABLE, null, "whatever", conf, out, err); + Assert.assertEquals("mk: 1" + lsep, outStr.toString()); + } +} diff --git metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java index ca1582e..0885098 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java @@ -52,8 +52,14 @@ import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hive.common.util.BloomFilter; +import org.apache.thrift.TBase; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TSimpleJSONProtocol; +import org.apache.thrift.transport.TMemoryBuffer; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; @@ -65,6 +71,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.NavigableMap; import java.util.Set; @@ -73,28 +80,28 @@ */ public class HBaseReadWrite { - @VisibleForTesting final static String AGGR_STATS_TABLE = "HBMS_AGGR_STATS"; - @VisibleForTesting final static String DB_TABLE = "HBMS_DBS"; - @VisibleForTesting final static String FUNC_TABLE = "HBMS_FUNCS"; - @VisibleForTesting final static String GLOBAL_PRIVS_TABLE = "HBMS_GLOBAL_PRIVS"; - @VisibleForTesting final static String PART_TABLE = "HBMS_PARTITIONS"; - @VisibleForTesting final static String ROLE_TABLE = "HBMS_ROLES"; - @VisibleForTesting final static String SD_TABLE = "HBMS_SDS"; - @VisibleForTesting final static String SECURITY_TABLE = "HBMS_SECURITY"; - @VisibleForTesting final static String SEQUENCES_TABLE = "HBMS_SEQUENCES"; - @VisibleForTesting final static String TABLE_TABLE = "HBMS_TBLS"; - @VisibleForTesting final static String USER_TO_ROLE_TABLE = "HBMS_USER_TO_ROLE"; - @VisibleForTesting final static byte[] CATALOG_CF = "c".getBytes(HBaseUtils.ENCODING); - @VisibleForTesting final static byte[] STATS_CF = "s".getBytes(HBaseUtils.ENCODING); - @VisibleForTesting final static String NO_CACHE_CONF = "no.use.cache"; + final static String AGGR_STATS_TABLE = "HBMS_AGGR_STATS"; + final static String DB_TABLE = "HBMS_DBS"; + final static String FUNC_TABLE = "HBMS_FUNCS"; + final static String GLOBAL_PRIVS_TABLE = "HBMS_GLOBAL_PRIVS"; + final static String PART_TABLE = "HBMS_PARTITIONS"; + final static String ROLE_TABLE = "HBMS_ROLES"; + final static String SD_TABLE = "HBMS_SDS"; + final static String SECURITY_TABLE = "HBMS_SECURITY"; + final static String SEQUENCES_TABLE = "HBMS_SEQUENCES"; + final static String TABLE_TABLE = "HBMS_TBLS"; + final static String USER_TO_ROLE_TABLE = "HBMS_USER_TO_ROLE"; + final static byte[] CATALOG_CF = "c".getBytes(HBaseUtils.ENCODING); + final static byte[] STATS_CF = "s".getBytes(HBaseUtils.ENCODING); + final static String NO_CACHE_CONF = "no.use.cache"; /** * List of tables in HBase */ - public final static String[] tableNames = { AGGR_STATS_TABLE, DB_TABLE, FUNC_TABLE, GLOBAL_PRIVS_TABLE, - PART_TABLE, USER_TO_ROLE_TABLE, ROLE_TABLE, SD_TABLE, - SECURITY_TABLE, SEQUENCES_TABLE, TABLE_TABLE}; - public final static Map> columnFamilies = - new HashMap> (tableNames.length); + public final static String[] tableNames = { AGGR_STATS_TABLE, DB_TABLE, FUNC_TABLE, + GLOBAL_PRIVS_TABLE, PART_TABLE, USER_TO_ROLE_TABLE, + ROLE_TABLE, SD_TABLE, SECURITY_TABLE, SEQUENCES_TABLE, + TABLE_TABLE}; + public final static Map> columnFamilies = new HashMap<> (tableNames.length); static { columnFamilies.put(AGGR_STATS_TABLE, Arrays.asList(CATALOG_CF)); @@ -110,18 +117,14 @@ columnFamilies.put(TABLE_TABLE, Arrays.asList(CATALOG_CF, STATS_CF)); } - /** - * Stores the bloom filter for the aggregated stats, to determine what partitions are in this - * aggregate. - */ - final static byte[] MASTER_KEY_SEQUENCE = "mk".getBytes(HBaseUtils.ENCODING); final static byte[] AGGR_STATS_BLOOM_COL = "b".getBytes(HBaseUtils.ENCODING); + private final static byte[] AGGR_STATS_STATS_COL = "s".getBytes(HBaseUtils.ENCODING); + final static byte[] MASTER_KEY_SEQUENCE = "mk".getBytes(HBaseUtils.ENCODING); private final static byte[] CATALOG_COL = "c".getBytes(HBaseUtils.ENCODING); private final static byte[] ROLES_COL = "roles".getBytes(HBaseUtils.ENCODING); private final static byte[] REF_COUNT_COL = "ref".getBytes(HBaseUtils.ENCODING); private final static byte[] DELEGATION_TOKEN_COL = "dt".getBytes(HBaseUtils.ENCODING); private final static byte[] MASTER_KEY_COL = "mk".getBytes(HBaseUtils.ENCODING); - private final static byte[] AGGR_STATS_STATS_COL = "s".getBytes(HBaseUtils.ENCODING); private final static byte[] GLOBAL_PRIVS_KEY = "gp".getBytes(HBaseUtils.ENCODING); private final static byte[] SEQUENCES_KEY = "seq".getBytes(HBaseUtils.ENCODING); private final static int TABLES_TO_CACHE = 10; @@ -373,6 +376,35 @@ void deleteDb(String name) throws IOException { delete(DB_TABLE, key, null, null); } + /** + * Print out the database. Intended for use by {@link org.apache.hadoop.hive.metastore.hbase.HBaseSchemaTool} + * @param name name of database to print + * @return string printout of database + */ + String printDatabase(String name) throws IOException, TException { + Database db = getDb(name); + if (db == null) return noSuch(name, "database"); + else return dumpThriftObject(db); + } + + /** + * Print out databases. + * @param regex regular to use to search for databases + * @return databases as a string, one each + * @throws IOException + * @throws TException + */ + List printDatabases(String regex) throws IOException, TException { + List dbs = scanDatabases(regex); + if (dbs.size() == 0) { + return noMatch(regex, "database"); + } else { + List lines = new ArrayList<>(); + for (Database db : dbs) lines.add(dumpThriftObject(db)); + return lines; + } + } + /********************************************************************************************** * Function related methods *********************************************************************************************/ @@ -440,6 +472,41 @@ void deleteFunction(String dbName, String functionName) throws IOException { delete(FUNC_TABLE, key, null, null); } + /** + * Print out a function + * @param key key to get the function, must include dbname. + * @return string of the function + * @throws IOException + * @throws TException + */ + String printFunction(String key) throws IOException, TException { + byte[] k = HBaseUtils.buildKey(key); + byte[] serialized = read(FUNC_TABLE, k, CATALOG_CF, CATALOG_COL); + if (serialized == null) return noSuch(key, "function"); + Function func = HBaseUtils.deserializeFunction(k, serialized); + return dumpThriftObject(func); + } + + /** + * Print out functions + * @param regex regular expression to use in matching functions + * @return list of strings, one function each + * @throws IOException + * @throws TException + */ + List printFunctions(String regex) throws IOException, TException { + Filter filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex)); + Iterator iter = scan(FUNC_TABLE, null, null, CATALOG_CF, CATALOG_COL, filter); + List lines = new ArrayList<>(); + while (iter.hasNext()) { + Result result = iter.next(); + lines.add(dumpThriftObject(HBaseUtils.deserializeFunction(result.getRow(), + result.getValue(CATALOG_CF, CATALOG_COL)))); + } + if (lines.size() == 0) lines = noMatch(regex, "function"); + return lines; + } + /********************************************************************************************** * Global privilege related methods *********************************************************************************************/ @@ -466,6 +533,18 @@ void putGlobalPrivs(PrincipalPrivilegeSet privs) throws IOException { store(GLOBAL_PRIVS_TABLE, key, CATALOG_CF, CATALOG_COL, serialized); } + /** + * Print out the global privileges. + * @return string containing the global privileges + * @throws IOException + * @throws TException + */ + String printGlobalPrivs() throws IOException, TException { + PrincipalPrivilegeSet pps = getGlobalPrivs(); + if (pps == null) return "No global privileges"; + else return dumpThriftObject(pps); + } + /********************************************************************************************** * Partition related methods *********************************************************************************************/ @@ -625,7 +704,8 @@ void replacePartitions(List oldParts, List newParts) throw : new ArrayList<>(cached); } byte[] keyPrefix = HBaseUtils.buildKeyWithTrailingSeparator(dbName, tableName); - List parts = scanPartitionsWithFilter(keyPrefix, HBaseUtils.getEndPrefix(keyPrefix), -1, null); + List parts = scanPartitionsWithFilter(keyPrefix, HBaseUtils.getEndPrefix(keyPrefix), + -1, null); partCache.put(dbName, tableName, parts, true); return maxPartitions < parts.size() ? parts.subList(0, maxPartitions) : parts; } @@ -723,7 +803,8 @@ void replacePartitions(List oldParts, List newParts) throw keyElements.add(tableName); byte[] keyPrefix = - HBaseUtils.buildKeyWithTrailingSeparator(keyElements.toArray(new String[keyElements.size()])); + HBaseUtils.buildKeyWithTrailingSeparator( + keyElements.toArray(new String[keyElements.size()])); byte[] startRow = ArrayUtils.addAll(keyPrefix, keyStart); byte[] endRow; if (keyEnd == null || keyEnd.length == 0) { @@ -740,8 +821,6 @@ void replacePartitions(List oldParts, List newParts) throw return scanPartitionsWithFilter(startRow, endRow, maxPartitions, filter); } - - /** * Delete a partition * @param dbName database name that table is in @@ -754,6 +833,73 @@ void deletePartition(String dbName, String tableName, List partVals) thr deletePartition(dbName, tableName, partVals, true); } + /** + * Print out a partition. + * @param partKey The key for the partition. This must include dbname.tablename._partkeys_ + * where _partkeys_ is a dot separated list of partition values in the proper + * order. + * @return string containing the partition + * @throws IOException + * @throws TException + */ + String printPartition(String partKey) throws IOException, TException { + byte[] key = HBaseUtils.buildKey(partKey); + @SuppressWarnings("deprecation") + HTableInterface htab = conn.getHBaseTable(PART_TABLE); + Get g = new Get(key); + g.addColumn(CATALOG_CF, CATALOG_COL); + g.addFamily(STATS_CF); + Result result = htab.get(g); + if (result.isEmpty()) return noSuch(partKey, "partition"); + return printOnePartition(result); + } + + /** + * Print partitions + * @param regex to use to find the partition. Remember that dbname.tablename is in each + * partition key. + * @return partitions as strings + * @throws IOException + * @throws TException + */ + List printPartitions(String regex) throws IOException, TException { + Filter filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex)); + @SuppressWarnings("deprecation") + HTableInterface htab = conn.getHBaseTable(PART_TABLE); + Scan scan = new Scan(); + scan.addColumn(CATALOG_CF, CATALOG_COL); + scan.addFamily(STATS_CF); + scan.setFilter(filter); + Iterator iter = htab.getScanner(scan).iterator(); + if (!iter.hasNext()) return noMatch(regex, "partition"); + List lines = new ArrayList<>(); + while (iter.hasNext()) { + lines.add(printOnePartition(iter.next())); + } + return lines; + } + + private String printOnePartition(Result result) throws IOException, TException { + byte[] key = result.getRow(); + HBaseUtils.StorageDescriptorParts sdParts = + HBaseUtils.deserializePartition(key, result.getValue(CATALOG_CF, CATALOG_COL)); + StringBuilder builder = new StringBuilder(); + builder.append(dumpThriftObject(sdParts.containingPartition)) + .append(" sdHash: ") + .append(Base64.encodeBase64URLSafeString(sdParts.sdHash)) + .append(" stats:"); + NavigableMap statsCols = result.getFamilyMap(STATS_CF); + for (Map.Entry statsCol : statsCols.entrySet()) { + builder.append(" column ") + .append(new String(statsCol.getKey(), HBaseUtils.ENCODING)) + .append(": "); + ColumnStatistics pcs = buildColStats(key, null); + ColumnStatisticsObj cso = HBaseUtils.deserializeStatsForOneColumn(pcs, statsCol.getValue()); + builder.append(dumpThriftObject(cso)); + } + return builder.toString(); + } + private void deletePartition(String dbName, String tableName, List partVals, boolean decrementRefCnt) throws IOException { // Find the partition so I can get the storage descriptor and drop it @@ -1132,14 +1278,7 @@ Role getRole(String roleName) throws IOException { * @throws IOException */ List scanRoles() throws IOException { - Iterator iter = scan(ROLE_TABLE, CATALOG_CF, CATALOG_COL); - List roles = new ArrayList<>(); - while (iter.hasNext()) { - Result result = iter.next(); - roles.add(HBaseUtils.deserializeRole(result.getRow(), - result.getValue(CATALOG_CF, CATALOG_COL))); - } - return roles; + return scanRoles(null); } /** @@ -1163,6 +1302,70 @@ void deleteRole(String roleName) throws IOException { roleCache.remove(roleName); } + String printRolesForUser(String userName) throws IOException { + List roles = getUserRoles(userName); + if (roles == null || roles.size() == 0) return noSuch(userName, "user"); + return org.apache.commons.lang.StringUtils.join(roles, ','); + } + + List printRolesForUsers(String regex) throws IOException { + Filter filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex)); + Iterator iter = scan(USER_TO_ROLE_TABLE, null, null, CATALOG_CF, CATALOG_COL, filter); + List lines = new ArrayList<>(); + while (iter.hasNext()) { + Result result = iter.next(); + lines.add(new String(result.getRow(), HBaseUtils.ENCODING) + ": " + + org.apache.commons.lang.StringUtils.join( + HBaseUtils.deserializeRoleList(result.getValue(CATALOG_CF, CATALOG_COL)), ',')); + } + if (lines.size() == 0) lines = noMatch(regex, "user"); + return lines; + } + + /** + * Print out a role + * @param name name of role to print + * @return string printout of role + */ + String printRole(String name) throws IOException, TException { + Role role = getRole(name); + if (role == null) return noSuch(name, "role"); + else return dumpThriftObject(role); + } + + /** + * Print out roles. + * @param regex regular to use to search for roles + * @return string printout of roles + * @throws IOException + * @throws TException + */ + List printRoles(String regex) throws IOException, TException { + List roles = scanRoles(regex); + if (roles.size() == 0) { + return noMatch(regex, "role"); + } else { + List lines = new ArrayList<>(); + for (Role role : roles) lines.add(dumpThriftObject(role)); + return lines; + } + } + + private List scanRoles(String regex) throws IOException { + Filter filter = null; + if (regex != null) { + filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex)); + } + Iterator iter = scan(ROLE_TABLE, null, null, CATALOG_CF, CATALOG_COL, filter); + List roles = new ArrayList<>(); + while (iter.hasNext()) { + Result result = iter.next(); + roles.add(HBaseUtils.deserializeRole(result.getRow(), + result.getValue(CATALOG_CF, CATALOG_COL))); + } + return roles; + } + private void buildRoleCache() throws IOException { if (!entireRoleTableInCache) { Iterator roles = scan(ROLE_TABLE, CATALOG_CF, ROLES_COL); @@ -1324,6 +1527,71 @@ void deleteTable(String dbName, String tableName) throws IOException { deleteTable(dbName, tableName, true); } + /** + * Print out a table. + * @param name The name for the table. This must include dbname.tablename + * @return string containing the table + * @throws IOException + * @throws TException + */ + String printTable(String name) throws IOException, TException { + byte[] key = HBaseUtils.buildKey(name); + @SuppressWarnings("deprecation") + HTableInterface htab = conn.getHBaseTable(TABLE_TABLE); + Get g = new Get(key); + g.addColumn(CATALOG_CF, CATALOG_COL); + g.addFamily(STATS_CF); + Result result = htab.get(g); + if (result.isEmpty()) return noSuch(name, "table"); + return printOneTable(result); + } + + /** + * Print tables + * @param regex to use to find the tables. Remember that dbname is in each + * table name. + * @return tables as strings + * @throws IOException + * @throws TException + */ + List printTables(String regex) throws IOException, TException { + Filter filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex)); + @SuppressWarnings("deprecation") + HTableInterface htab = conn.getHBaseTable(TABLE_TABLE); + Scan scan = new Scan(); + scan.addColumn(CATALOG_CF, CATALOG_COL); + scan.addFamily(STATS_CF); + scan.setFilter(filter); + Iterator iter = htab.getScanner(scan).iterator(); + if (!iter.hasNext()) return noMatch(regex, "table"); + List lines = new ArrayList<>(); + while (iter.hasNext()) { + lines.add(printOneTable(iter.next())); + } + return lines; + } + + private String printOneTable(Result result) throws IOException, TException { + byte[] key = result.getRow(); + HBaseUtils.StorageDescriptorParts sdParts = + HBaseUtils.deserializeTable(key, result.getValue(CATALOG_CF, CATALOG_COL)); + StringBuilder builder = new StringBuilder(); + builder.append(dumpThriftObject(sdParts.containingTable)) + .append(" sdHash: ") + .append(Base64.encodeBase64URLSafeString(sdParts.sdHash)) + .append(" stats:"); + NavigableMap statsCols = result.getFamilyMap(STATS_CF); + for (Map.Entry statsCol : statsCols.entrySet()) { + builder.append(" column ") + .append(new String(statsCol.getKey(), HBaseUtils.ENCODING)) + .append(": "); + ColumnStatistics pcs = buildColStats(key, null); + ColumnStatisticsObj cso = HBaseUtils.deserializeStatsForOneColumn(pcs, statsCol.getValue()); + builder.append(dumpThriftObject(cso)); + } + return builder.toString(); + } + private void deleteTable(String dbName, String tableName, boolean decrementRefCnt) throws IOException { tableCache.remove(new ObjectPair<>(dbName, tableName)); @@ -1440,6 +1708,37 @@ void decrementStorageDescriptorRefCount(StorageDescriptor sd) throws IOException return key; } + /** + * Print out a storage descriptor. + * @param hash hash that is the key of the storage descriptor + * @return string version of the storage descriptor + */ + String printStorageDescriptor(byte[] hash) throws IOException, TException { + byte[] serialized = read(SD_TABLE, hash, CATALOG_CF, CATALOG_COL); + if (serialized == null) return noSuch(Base64.encodeBase64URLSafeString(hash), "storage descriptor"); + return dumpThriftObject(HBaseUtils.deserializeStorageDescriptor(serialized)); + } + + /** + * Print all of the storage descriptors. This doesn't take a regular expression since the key + * is an md5 hash and it's hard to see how a regex on this would be useful. + * @return list of all storage descriptors as strings + * @throws IOException + * @throws TException + */ + List printStorageDescriptors() throws IOException, TException { + Iterator results = scan(SD_TABLE, CATALOG_CF, CATALOG_COL); + if (!results.hasNext()) return Arrays.asList("No storage descriptors"); + List lines = new ArrayList<>(); + while (results.hasNext()) { + Result result = results.next(); + lines.add(Base64.encodeBase64URLSafeString(result.getRow()) + ": " + + dumpThriftObject(HBaseUtils.deserializeStorageDescriptor(result.getValue(CATALOG_CF, + CATALOG_COL)))); + } + return lines; + } + private static class ByteArrayWrapper { byte[] wrapped; @@ -1576,24 +1875,8 @@ ColumnStatistics getTableStatistics(String dbName, String tblName, List if (colStats == null) { // We initialize this late so that we don't create extras in the case of // partitions with no stats - colStats = new ColumnStatistics(); + colStats = buildColStats(results[i].getRow(), valToPartMap); statsList.add(colStats); - ColumnStatisticsDesc csd = new ColumnStatisticsDesc(); - - // We need to figure out which partition these call stats are from. To do that we - // recontruct the key. We have to pull the dbName and tableName out of the key to - // find the partition values. - byte[] key = results[i].getRow(); - String[] reconstructedKey = HBaseUtils.parseKey(key); - List reconstructedPartVals = - Arrays.asList(reconstructedKey).subList(2, reconstructedKey.length); - String partName = valToPartMap.get(reconstructedPartVals); - assert partName != null; - csd.setIsTblLevel(false); - csd.setDbName(dbName); - csd.setTableName(tblName); - csd.setPartName(partName); - colStats.setStatsDesc(csd); } ColumnStatisticsObj cso = HBaseUtils.deserializeStatsForOneColumn(colStats, serializedColStats); @@ -1699,6 +1982,27 @@ private String getStatisticsTable(List partVals) { return partVals == null ? TABLE_TABLE : PART_TABLE; } + private ColumnStatistics buildColStats(byte[] key, Map, String> valToPartMap) { + // We initialize this late so that we don't create extras in the case of + // partitions with no stats + ColumnStatistics colStats = new ColumnStatistics(); + ColumnStatisticsDesc csd = new ColumnStatisticsDesc(); + + // We need to figure out which partition these call stats are from. To do that we + // recontruct the key. We have to pull the dbName and tableName out of the key to + // find the partition values. + String[] reconstructedKey = HBaseUtils.parseKey(key); + List reconstructedPartVals = + Arrays.asList(reconstructedKey).subList(2, reconstructedKey.length); + String partName = null; + if (valToPartMap != null) partName = valToPartMap.get(reconstructedPartVals); + csd.setIsTblLevel(false); + csd.setDbName(reconstructedKey[0]); + csd.setTableName(reconstructedKey[1]); + if (partName != null) csd.setPartName(partName); + colStats.setStatsDesc(csd); + return colStats; + } /********************************************************************************************** * Security related methods *********************************************************************************************/ @@ -1812,6 +2116,35 @@ void deleteMasterKey(Integer seqNo) throws IOException { delete(SECURITY_TABLE, key, CATALOG_CF, MASTER_KEY_COL); } + /** + * One method to print all rows in the security table. It's not expected to be large. + * @return each row as one string + * @throws IOException + */ + List printSecurity() throws IOException { + HTableInterface htab = conn.getHBaseTable(SECURITY_TABLE); + Scan scan = new Scan(); + scan.addColumn(CATALOG_CF, MASTER_KEY_COL); + scan.addColumn(CATALOG_CF, DELEGATION_TOKEN_COL); + Iterator iter = htab.getScanner(scan).iterator(); + if (!iter.hasNext()) return Arrays.asList("No security related entries"); + List lines = new ArrayList<>(); + while (iter.hasNext()) { + Result result = iter.next(); + byte[] val = result.getValue(CATALOG_CF, MASTER_KEY_COL); + if (val != null) { + int seqNo = Integer.valueOf(new String(result.getRow(), HBaseUtils.ENCODING)); + lines.add("Master key " + seqNo + ": " + HBaseUtils.deserializeMasterKey(val)); + } else { + val = result.getValue(CATALOG_CF, DELEGATION_TOKEN_COL); + if (val == null) throw new RuntimeException("Huh? No master key, no delegation token!"); + lines.add("Delegation token " + new String(result.getRow(), HBaseUtils.ENCODING) + ": " + + HBaseUtils.deserializeDelegationToken(val)); + } + } + return lines; + } + /********************************************************************************************** * Sequence methods *********************************************************************************************/ @@ -1827,6 +2160,25 @@ long getNextSequence(byte[] sequence) throws IOException { return val; } + /** + * One method to print all entries in the sequence table. It's not expected to be large. + * @return each sequence as one string + * @throws IOException + */ + List printSequences() throws IOException { + HTableInterface htab = conn.getHBaseTable(SEQUENCES_TABLE); + Get g = new Get(SEQUENCES_KEY); + g.addFamily(CATALOG_CF); + Result result = htab.get(g); + if (result.isEmpty()) return Arrays.asList("No sequences"); + List lines = new ArrayList<>(); + for (Map.Entry entry : result.getFamilyMap(CATALOG_CF).entrySet()) { + lines.add(new String(entry.getKey(), HBaseUtils.ENCODING) + ": " + + new String(entry.getValue(), HBaseUtils.ENCODING)); + } + return lines; + } + /********************************************************************************************** * Cache methods *********************************************************************************************/ @@ -1937,12 +2289,27 @@ private void delete(String table, byte[] key, byte[] colFam, byte[] colName) thr return scanner.iterator(); } + /********************************************************************************************** + * Printing methods + *********************************************************************************************/ + private String noSuch(String name, String type) { + return "No such " + type + ": " + name; + } + private List noMatch(String regex, String type) { + return Arrays.asList("No matching " + type + ": " + regex); + } + + private String dumpThriftObject(TBase obj) throws TException, UnsupportedEncodingException { + TMemoryBuffer buf = new TMemoryBuffer(1000); + TProtocol protocol = new TSimpleJSONProtocol(buf); + obj.write(protocol); + return buf.toString("UTF-8"); + } /********************************************************************************************** * Testing methods and classes *********************************************************************************************/ - @VisibleForTesting int countStorageDescriptor() throws IOException { ResultScanner scanner = conn.getHBaseTable(SD_TABLE).getScanner(new Scan()); diff --git metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java index 1c407f1..22777a1 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java @@ -18,27 +18,20 @@ */ package org.apache.hadoop.hive.metastore.hbase; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.cli.ParseException; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.api.ColumnStatistics; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.Function; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.Role; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.thrift.TBase; -import org.apache.thrift.TException; -import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.protocol.TSimpleJSONProtocol; -import org.apache.thrift.transport.TMemoryBuffer; - -import java.io.IOException; -import java.lang.reflect.Method; + +import java.io.PrintStream; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -47,193 +40,133 @@ */ public class HBaseSchemaTool { - private static String[] commands = {"db", "part", "parts", "role", "table", "function", - "install"}; + static final private Log LOG = LogFactory.getLog(HBaseReadWrite.class.getName()); - public static void main(String[] args) throws Exception { + public static void main(String[] args) { Options options = new Options(); options.addOption(OptionBuilder - .withLongOpt("column") - .withDescription("Comma separated list of column names") - .hasArg() - .create('c')); - - options.addOption(OptionBuilder - .withLongOpt("db") - .withDescription("Database name") - .hasArg() - .create('d')); - - options.addOption(OptionBuilder - .withLongOpt("function") - .withDescription("Function name") - .hasArg() - .create('f')); - - options.addOption(OptionBuilder .withLongOpt("help") .withDescription("You're looking at it") .create('h')); options.addOption(OptionBuilder - .withLongOpt("role") - .withDescription("Role name") + .withLongOpt("key") + .withDescription("Key to scan with. This should be an exact key (not a regular expression") .hasArg() - .create('r')); + .create('k')); options.addOption(OptionBuilder - .withLongOpt("partvals") - .withDescription("Comma separated list of partition values, in order of partition columns") + .withLongOpt("list-tables") + .withDescription("List tables in HBase metastore") .hasArg() - .create('p')); + .create('l')); options.addOption(OptionBuilder - .withLongOpt("stats") - .withDescription("Get statistics rather than catalog object") - .create('s')); + .withLongOpt("regex-key") + .withDescription("Regular expression to scan keys with.") + .hasArg() + .create('r')); options.addOption(OptionBuilder .withLongOpt("table") - .withDescription("Table name") + .withDescription("HBase metastore table to scan") .hasArg() + .isRequired() .create('t')); - CommandLine cli = new GnuParser().parse(options, args); - - if (cli.hasOption('h')) { - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("hbaseschematool", options); + CommandLine cli = null; + try { + cli = new GnuParser().parse(options, args); + } catch (ParseException e) { + usage(options); return; } - String[] cmds = cli.getArgs(); - if (cmds.length != 1) { - System.err.print("Must include a cmd, valid cmds are: "); - for (int i = 0; i < commands.length; i++) { - if (i != 0) System.err.print(", "); - System.err.print(commands[i]); - } - System.err.println(); - System.exit(1); - } - String cmd = cmds[0]; - - List parts = null; - if (cli.hasOption('p')) { - parts = Arrays.asList(cli.getOptionValue('p').split(",")); + if (cli.hasOption('h')) { + usage(options); + return; } - List cols = null; - if (cli.hasOption('c')) { - cols = Arrays.asList(cli.getOptionValue('c').split(",")); + String key = null; + if (cli.hasOption('k')) key = cli.getOptionValue('k'); + String regex = null; + if (cli.hasOption('r')) regex = cli.getOptionValue('r'); + if (key != null && regex != null) { + usage(options); + return; } + if (key == null && regex == null) regex = ".*"; - HBaseSchemaTool tool = new HBaseSchemaTool(cli.getOptionValue('d'), cli.getOptionValue('t'), - parts, cli.getOptionValue('f'), cli.getOptionValue('r'), cols, cli.hasOption('s')); - Method method = tool.getClass().getMethod(cmd); - method.invoke(tool); - - - } - - private HBaseReadWrite hrw; - private String dbName; - private String funcName; - private String tableName; - private List partVals; - private String roleName; - private List colNames; - private boolean hasStats; - - private HBaseSchemaTool(String dbname, String tn, List pv, String fn, String rn, - List cn, boolean s) { - dbName = dbname; - tableName = tn; - partVals = pv; - funcName = fn; - roleName = rn; - colNames = cn; - hasStats = s; - hrw = HBaseReadWrite.getInstance(new Configuration()); - } + // I do this in the object rather than in the static main so that it's easier to test. + new HBaseSchemaTool().go(cli.hasOption('l'), cli.getOptionValue('t'), key, regex, new + Configuration(), System.out, System.err); - public void db() throws IOException, TException { - Database db = hrw.getDb(dbName); - if (db == null) System.err.println("No such database: " + db); - else dump(db); } - public void install() throws IOException { - HBaseReadWrite.createTablesIfNotExist(); + private static void usage(Options options) { + HelpFormatter formatter = new HelpFormatter(); + String header = "This tool dumps contents of your hbase metastore. You need to specify\n" + + "the table to dump. You can optionally specify a regular expression on the key for\n" + + "the table. Keep in mind that the key is often a compound. For partitions regexs\n" + + "on partition keys that are not strings is difficult because non-string values are\n" + + "stored in binary."; + String footer = "If neither key or regex is provided a regex of .* will be assumed. You\n" + + "cannot set both key and regex."; + formatter.printHelp("hbaseschematool", header, options, footer); + return; } - public void part() throws IOException, TException { - if (hasStats) { - Table table = hrw.getTable(dbName, tableName); - if (table == null) { - System.err.println("No such table: " + dbName + "." + tableName); - return; - } - String partName = HBaseStore.buildExternalPartName(table, partVals); - List stats = hrw.getPartitionStatistics(dbName, tableName, - Arrays.asList(partName), Arrays.asList(partVals), colNames); - if (stats == null) { - System.err.println("No stats for " + dbName + "." + tableName + "." + - StringUtils.join(partVals, ':')); - } else { - for (ColumnStatistics stat : stats) dump(stat); - } + @VisibleForTesting void go(boolean listTables, String table, String key, String regex, + Configuration conf, PrintStream out, PrintStream err) { + List lines = new ArrayList<>(); + if (listTables) { + lines = Arrays.asList(HBaseReadWrite.tableNames); } else { - Partition part = hrw.getPartition(dbName, tableName, partVals); - if (part == null) { - System.err.println("No such partition: " + dbName + "." + tableName + "." + - StringUtils.join(partVals, ':')); - } else { - dump(part); + // If they've used '.' as a key separator we need to replace it with the separator used by + // HBaseUtils + if (key != null) key = key.replace('.', HBaseUtils.KEY_SEPARATOR); + try { + HBaseReadWrite hrw = HBaseReadWrite.getInstance(conf); + if (table.equalsIgnoreCase(HBaseReadWrite.DB_TABLE)) { + if (key != null) lines.add(hrw.printDatabase(key)); + else lines.addAll(hrw.printDatabases(regex)); + } else if (table.equalsIgnoreCase(HBaseReadWrite.FUNC_TABLE)) { + if (key != null) lines.add(hrw.printFunction(key)); + else lines.addAll(hrw.printFunctions(regex)); + } else if (table.equalsIgnoreCase(HBaseReadWrite.GLOBAL_PRIVS_TABLE)) { + // Ignore whatever they passed, there's always only either one or zero global privileges + lines.add(hrw.printGlobalPrivs()); + } else if (table.equalsIgnoreCase(HBaseReadWrite.PART_TABLE)) { + if (key != null) lines.add(hrw.printPartition(key)); + else lines.addAll(hrw.printPartitions(regex)); + } else if (table.equalsIgnoreCase(HBaseReadWrite.USER_TO_ROLE_TABLE)) { + if (key != null) lines.add(hrw.printRolesForUser(key)); + else lines.addAll(hrw.printRolesForUsers(regex)); + } else if (table.equalsIgnoreCase(HBaseReadWrite.ROLE_TABLE)) { + if (key != null) lines.add(hrw.printRole(key)); + else lines.addAll(hrw.printRoles(regex)); + } else if (table.equalsIgnoreCase(HBaseReadWrite.TABLE_TABLE)) { + if (key != null) lines.add(hrw.printTable(key)); + else lines.addAll(hrw.printTables(regex)); + } else if (table.equalsIgnoreCase(HBaseReadWrite.SD_TABLE)) { + if (key != null) lines.add(hrw.printStorageDescriptor(Base64.decodeBase64(key))); + else lines.addAll(hrw.printStorageDescriptors()); + } else if (table.equalsIgnoreCase(HBaseReadWrite.SECURITY_TABLE)) { + // We always print all of security, we don't worry about finding particular entries. + lines.addAll(hrw.printSecurity()); + } else if (table.equalsIgnoreCase(HBaseReadWrite.SEQUENCES_TABLE)) { + // We always print all of sequences, we don't worry about finding particular entries. + lines.addAll(hrw.printSequences()); + } else { + err.println("Unknown table: " + table); + return; + } + } catch (Exception e) { + err.println("Caught exception " + e.getClass() + " with message: " + e.getMessage()); + return; } } + for (String line : lines) out.println(line); } - - public void parts() throws IOException, TException { - List parts = hrw.scanPartitionsInTable(dbName, tableName, -1); - if (parts == null) { - System.err.println("No such table: " + dbName + "." + tableName); - } else { - for (Partition p : parts) dump(p); - } - } - - public void role() throws IOException, TException { - Role role = hrw.getRole(roleName); - if (role == null) System.err.println("No such role: " + roleName); - else dump(role); - } - - public void table() throws IOException, TException { - if (hasStats) { - ColumnStatistics stats = hrw.getTableStatistics(dbName, tableName, colNames); - if (stats == null) System.err.println("No stats for " + dbName + "." + tableName); - else dump(stats); - } else { - Table table = hrw.getTable(dbName, tableName); - if (table == null) System.err.println("No such table: " + dbName + "." + tableName); - else dump(table); - } - } - - public void function() throws IOException, TException { - Function func = hrw.getFunction(dbName, funcName); - if (func == null) System.err.println("No such function: " + dbName + "." + funcName); - else dump(func); - } - - private void dump(TBase thriftObj) throws TException { - TMemoryBuffer buf = new TMemoryBuffer(1000); - TProtocol protocol = new TSimpleJSONProtocol(buf); - thriftObj.write(protocol); - System.out.println(new String(buf.getArray())); - } - - }