From c23d90b2ad48b6a2bd3d56a438ebd09cf9c21e9b Mon Sep 17 00:00:00 2001 From: Josh Elser Date: Fri, 24 Mar 2017 14:00:13 -0400 Subject: [PATCH] HBASE-17752 Shell command to list snapshot sizes WRT quotas --- .../apache/hadoop/hbase/quotas/QuotaTableUtil.java | 61 ++++++++++++++++++---- .../hbase/quotas/TestSpaceQuotasWithSnapshots.java | 10 ++++ hbase-shell/src/main/ruby/hbase/quotas.rb | 4 ++ hbase-shell/src/main/ruby/shell.rb | 1 + .../ruby/shell/commands/list_snapshot_sizes.rb | 42 +++++++++++++++ 5 files changed, 108 insertions(+), 10 deletions(-) create mode 100644 hbase-shell/src/main/ruby/shell/commands/list_snapshot_sizes.rb diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java index c1863a7d63..71da1a1881 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java @@ -31,6 +31,7 @@ import java.util.regex.Pattern; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; @@ -513,16 +514,50 @@ public class QuotaTableUtil { return QuotaProtos.SpaceQuotaSnapshot.parseFrom(bs).getQuotaUsage(); } - static Scan createScanForSnapshotSizes(TableName table) { - byte[] rowkey = getTableRowKey(table); - return new Scan() - // Fetch just this one row - .withStartRow(rowkey) - .withStopRow(rowkey, true) - // Just the usage family - .addFamily(QUOTA_FAMILY_USAGE) - // Only the snapshot size qualifiers - .setFilter(new ColumnPrefixFilter(QUOTA_SNAPSHOT_SIZE_QUALIFIER)); + static Scan createScanForSpaceSnapshotSizes() { + return createScanForSpaceSnapshotSizes(null); + } + + public static Scan createScanForSpaceSnapshotSizes(TableName table) { + Scan s = new Scan(); + if (null == table) { + // Read all tables, just look at the row prefix + s.setRowPrefixFilter(QUOTA_TABLE_ROW_KEY_PREFIX); + } else { + // Fetch the exact row for the table + byte[] rowkey = getTableRowKey(table); + // Fetch just this one row + s.withStartRow(rowkey).withStopRow(rowkey, true); + } + + // Just the usage family and only the snapshot size qualifiers + return s.addFamily(QUOTA_FAMILY_USAGE).setFilter( + new ColumnPrefixFilter(QUOTA_SNAPSHOT_SIZE_QUALIFIER)); + } + + /** + * Fetches any persisted HBase snapshot sizes stored in the quota table. The sizes here are + * computed relative to the table which the snapshot was created from. A snapshot's size will + * not include the size of files which the table still refers. These sizes, in bytes, are what + * is used internally to compute quota violation for tables and namespaces. + * + * @return A map of snapshot name to size in bytes per space quota computations + */ + public static Map getObservedSnapshotSizes(Connection conn) throws IOException { + try (Table quotaTable = conn.getTable(QUOTA_TABLE_NAME); + ResultScanner rs = quotaTable.getScanner(createScanForSpaceSnapshotSizes())) { + final Map snapshotSizes = new HashMap<>(); + for (Result r : rs) { + CellScanner cs = r.cellScanner(); + while (cs.advance()) { + Cell c = cs.current(); + final String snapshot = extractSnapshotNameFromSizeCell(c); + final long size = parseSnapshotSize(c); + snapshotSizes.put(snapshot, size); + } + } + return snapshotSizes; + } } /* ========================================================================= @@ -749,6 +784,12 @@ public class QuotaTableUtil { return Bytes.add(QUOTA_SNAPSHOT_SIZE_QUALIFIER, Bytes.toBytes(snapshotName)); } + protected static String extractSnapshotNameFromSizeCell(Cell c) { + return Bytes.toString( + c.getQualifierArray(), c.getQualifierOffset() + QUOTA_SNAPSHOT_SIZE_QUALIFIER.length, + c.getQualifierLength() - QUOTA_SNAPSHOT_SIZE_QUALIFIER.length); + } + protected static long extractSnapshotSize( byte[] data, int offset, int length) throws InvalidProtocolBufferException { ByteString byteStr = UnsafeByteOperations.unsafeWrap(data, offset, length); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotasWithSnapshots.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotasWithSnapshots.java index ebb1a9e257..4c7d8bebd3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotasWithSnapshots.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotasWithSnapshots.java @@ -173,6 +173,11 @@ public class TestSpaceQuotasWithSnapshots { return expectedFinalSize == snapshot.getUsage(); } }); + + Map snapshotSizes = QuotaTableUtil.getObservedSnapshotSizes(conn); + Long size = snapshotSizes.get(snapshot1); + assertNotNull("Did not observe the size of the snapshot", size); + assertEquals(actualInitialSize, size.longValue()); } @Test @@ -265,6 +270,11 @@ public class TestSpaceQuotasWithSnapshots { return expectedFinalSize == snapshot.getUsage(); } }); + + Map snapshotSizes = QuotaTableUtil.getObservedSnapshotSizes(conn); + Long size = snapshotSizes.get(snapshot1); + assertNotNull("Did not observe the size of the snapshot", size); + assertEquals(actualInitialSize, size.longValue()); } @Test diff --git a/hbase-shell/src/main/ruby/hbase/quotas.rb b/hbase-shell/src/main/ruby/hbase/quotas.rb index 784896e730..a8a8e6ba80 100644 --- a/hbase-shell/src/main/ruby/hbase/quotas.rb +++ b/hbase-shell/src/main/ruby/hbase/quotas.rb @@ -241,6 +241,10 @@ module Hbase return count end + def list_snapshot_sizes() + QuotaTableUtil.getObservedSnapshotSizes(@admin.getConnection()) + end + def _parse_size(str_limit) str_limit = str_limit.downcase match = /(\d+)([bkmgtp%]*)/.match(str_limit) diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb index aaf26b36fe..ab4d5149d5 100644 --- a/hbase-shell/src/main/ruby/shell.rb +++ b/hbase-shell/src/main/ruby/shell.rb @@ -423,6 +423,7 @@ Shell.load_command_group( list_quotas list_quota_table_sizes list_quota_snapshots + list_snapshot_sizes ] ) diff --git a/hbase-shell/src/main/ruby/shell/commands/list_snapshot_sizes.rb b/hbase-shell/src/main/ruby/shell/commands/list_snapshot_sizes.rb new file mode 100644 index 0000000000..4ab8db9817 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/list_snapshot_sizes.rb @@ -0,0 +1,42 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListSnapshotSizes < Command + def help + return <<-EOF +Lists the size of every HBase snapshot given the space quota size computation +algorithms. An HBase snapshot only "owns" the size of a file when the table +from which the snapshot was created no longer refers to that file. +EOF + end + + def command(args = {}) + formatter.header(["SNAPSHOT", "SIZE"]) + count = 0 + quotas_admin.list_snapshot_sizes().each do |snapshot,size| + formatter.row([snapshot.to_s, size.to_s]) + count += 1 + end + formatter.footer(count) + end + end + end +end -- 2.12.2