Index: src/main/java/org/apache/hadoop/hbase/KeyValue.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/KeyValue.java (revision 1158066) +++ src/main/java/org/apache/hadoop/hbase/KeyValue.java (working copy) @@ -629,6 +629,7 @@ stringMap.put("family", Bytes.toStringBinary(getFamily())); stringMap.put("qualifier", Bytes.toStringBinary(getQualifier())); stringMap.put("timestamp", getTimestamp()); + stringMap.put("vlen", getValueLength()); return stringMap; } Index: src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java (revision 1158066) +++ src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java (working copy) @@ -35,7 +35,11 @@ import org.apache.commons.logging.*; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.client.DatabaseCommand; import org.apache.hadoop.hbase.io.HbaseObjectWritable; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Objects; import org.apache.hadoop.io.*; import org.apache.hadoop.ipc.VersionedProtocol; @@ -43,6 +47,9 @@ import org.apache.hadoop.security.authorize.ServiceAuthorizationManager; import org.apache.hadoop.conf.*; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; + /** An RpcEngine implementation for Writable data. */ class WritableRpcEngine implements RpcEngine { // LOG is NOT in hbase subpackage intentionally so that the default HBase @@ -246,6 +253,14 @@ private boolean verbose; private boolean authorize = false; + private static final String WARN_RESPONSE_TIME = + "hbase.ipc.warn.response.time"; + + /** Default value for above param */ + private static final int DEFAULT_WARN_RESPONSE_TIME = 1000; // milliseconds + + private final int warnResponseTime; + private static String classNameBase(String className) { String[] names = className.split("\\.", -1); if (names == null || names.length == 0) { @@ -282,6 +297,9 @@ this.authorize = conf.getBoolean( ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG, false); + + this.warnResponseTime = conf.getInt(WARN_RESPONSE_TIME, + DEFAULT_WARN_RESPONSE_TIME); } @Override @@ -321,6 +339,39 @@ " processingTime=" + processingTime + " contents=" + Objects.describeQuantity(params)); } + // log slow queries + if (processingTime > warnResponseTime) { + // if the slow process is a query, we want to log its table as well + // as its own fingerprint + if(params.length == 2 && instance instanceof HRegionServer && + params[0] instanceof byte[] && + params[1] instanceof DatabaseCommand) { + byte [] tableName = + HRegionInfo.parseRegionName((byte[]) params[0])[0]; + JSONObject json = ((DatabaseCommand) params[1]).toJSON(); + try { + json.put("table", Bytes.toStringBinary(tableName)); + json.put("processingtime", processingTime); + json.put("call", call.getMethodName()); + } catch (JSONException e) { + // ignore + } + LOG.warn("(queryTooSlow): " + json); + } else if (params.length == 1 && instance instanceof HRegionServer && + params[0] instanceof DatabaseCommand) { + JSONObject json = ((DatabaseCommand) params[0]).toJSON(); + try { + json.put("processingtime", processingTime); + json.put("call", call.getMethodName()); + } catch (JSONException e) { + // ignore + } + LOG.warn("(queryTooSlow): " + json); + } else { + LOG.warn("(responseTooSlow): Time (ms): " + processingTime + + " for call to " + call); + } + } rpcMetrics.rpcQueueTime.inc(qTime); rpcMetrics.rpcProcessingTime.inc(processingTime); rpcMetrics.inc(call.getMethodName(), processingTime); Index: src/main/java/org/apache/hadoop/hbase/client/Delete.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/client/Delete.java (revision 1158066) +++ src/main/java/org/apache/hadoop/hbase/client/Delete.java (working copy) @@ -68,7 +68,8 @@ * deleteFamily -- then you need to use the method overrides that take a * timestamp. The constructor timestamp is not referenced. */ -public class Delete implements Writable, Row, Comparable { +public class Delete extends DatabaseCommand + implements Writable, Row, Comparable { private static final byte DELETE_VERSION = (byte)3; private byte [] row = null; @@ -340,39 +341,67 @@ } /** - * @return string + * Compile the column family (i.e. schema) information + * into a Map. Useful for parsing and aggregation by debugging, + * logging, and administration tools. + * @return Map */ @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("row="); - sb.append(Bytes.toStringBinary(this.row)); - sb.append(", ts="); - sb.append(this.ts); - sb.append(", families={"); - boolean moreThanOne = false; + public Map getFingerprint() { + Map map = new HashMap(); + List families = new ArrayList(); + // ideally, we would also include table information, but that information + // is not stored in each DatabaseCommand instance. + map.put("families", families); for(Map.Entry> entry : this.familyMap.entrySet()) { - if(moreThanOne) { - sb.append(", "); - } else { - moreThanOne = true; + families.add(Bytes.toStringBinary(entry.getKey())); + } + return map; + } + + /** + * Compile the details beyond the scope of getFingerprint (row, columns, + * timestamps, etc.) into a Map along with the fingerprinted information. + * Useful for debugging, logging, and administration tools. + * @param maxCols a limit on the number of columns output prior to truncation + * @return Map + */ + @Override + public Map getDetails(int maxCols) { + // we start with the fingerprint map and build on top of it. + Map map = getFingerprint(); + // replace the fingerprint's simple list of families with a + // map from column families to lists of qualifiers and kv details + Map>> columns = + new HashMap>>(); + map.put("families", columns); + map.put("row", Bytes.toStringBinary(this.row)); + map.put("ts", this.ts); + int colCount = 0; + // iterate through all column families affected by this Delete + for(Map.Entry> entry : this.familyMap.entrySet()) { + // map from this family to details for each kv affected within the family + List> qualifierDetails = + new ArrayList>(); + columns.put(Bytes.toStringBinary(entry.getKey()), qualifierDetails); + colCount += entry.getValue().size(); + if (maxCols <= 0) { + continue; } - sb.append("(family="); - sb.append(Bytes.toString(entry.getKey())); - sb.append(", keyvalues=("); - boolean moreThanOneB = false; + // add details for each kv for(KeyValue kv : entry.getValue()) { - if(moreThanOneB) { - sb.append(", "); - } else { - moreThanOneB = true; + if (--maxCols <= 0 ) { + continue; } - sb.append(kv.toString()); + Map kvMap = kv.toStringMap(); + // row and family information are already available in the bigger map + kvMap.remove("row"); + kvMap.remove("family"); + qualifierDetails.add(kvMap); } - sb.append(")"); } - sb.append("}"); - return sb.toString(); + map.put("totalColumns", colCount); + return map; } //Writable Index: src/main/java/org/apache/hadoop/hbase/client/Put.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/client/Put.java (revision 1158066) +++ src/main/java/org/apache/hadoop/hbase/client/Put.java (working copy) @@ -47,7 +47,8 @@ * for each column to be inserted, execute {@link #add(byte[], byte[], byte[]) add} or * {@link #add(byte[], byte[], long, byte[]) add} if setting the timestamp. */ -public class Put implements HeapSize, Writable, Row, Comparable { +public class Put extends DatabaseCommand + implements HeapSize, Writable, Row, Comparable { private static final byte PUT_VERSION = (byte)2; private byte [] row = null; @@ -462,39 +463,67 @@ return Collections.unmodifiableMap(attributes); } - /** - * @return String + * Compile the column family (i.e. schema) information + * into a Map. Useful for parsing and aggregation by debugging, + * logging, and administration tools. + * @return Map */ @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("row="); - sb.append(Bytes.toStringBinary(this.row)); - sb.append(", families={"); - boolean moreThanOne = false; + public Map getFingerprint() { + Map map = new HashMap(); + List families = new ArrayList(); + // ideally, we would also include table information, but that information + // is not stored in each DatabaseCommand instance. + map.put("families", families); for(Map.Entry> entry : this.familyMap.entrySet()) { - if(moreThanOne) { - sb.append(", "); - } else { - moreThanOne = true; + families.add(Bytes.toStringBinary(entry.getKey())); + } + return map; + } + + /** + * Compile the details beyond the scope of getFingerprint (row, columns, + * timestamps, etc.) into a Map along with the fingerprinted information. + * Useful for debugging, logging, and administration tools. + * @param maxCols a limit on the number of columns output prior to truncation + * @return Map + */ + @Override + public Map getDetails(int maxCols) { + // we start with the fingerprint map and build on top of it. + Map map = getFingerprint(); + // replace the fingerprint's simple list of families with a + // map from column families to lists of qualifiers and kv details + Map>> columns = + new HashMap>>(); + map.put("families", columns); + map.put("row", Bytes.toStringBinary(this.row)); + int colCount = 0; + // iterate through all column families affected by this Put + for(Map.Entry> entry : this.familyMap.entrySet()) { + // map from this family to details for each kv affected within the family + List> qualifierDetails = + new ArrayList>(); + columns.put(Bytes.toStringBinary(entry.getKey()), qualifierDetails); + colCount += entry.getValue().size(); + if (maxCols <= 0) { + continue; } - sb.append("(family="); - sb.append(Bytes.toString(entry.getKey())); - sb.append(", keyvalues=("); - boolean moreThanOneB = false; - for(KeyValue kv : entry.getValue()) { - if(moreThanOneB) { - sb.append(", "); - } else { - moreThanOneB = true; - } - sb.append(kv.toString()); - } - sb.append(")"); - } - sb.append("}"); - return sb.toString(); + // add details for each kv + for(KeyValue kv : entry.getValue()) { + if (--maxCols <= 0 ) { + continue; + } + Map kvMap = kv.toStringMap(); + // row and family information are already available in the bigger map + kvMap.remove("row"); + kvMap.remove("family"); + qualifierDetails.add(kvMap); + } + } + map.put("totalColumns", colCount); + return map; } public int compareTo(Row p) { Index: src/main/java/org/apache/hadoop/hbase/client/Get.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/client/Get.java (revision 1158066) +++ src/main/java/org/apache/hadoop/hbase/client/Get.java (working copy) @@ -31,8 +31,10 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.NavigableSet; import java.util.Set; @@ -63,7 +65,8 @@ *

* To add a filter, execute {@link #setFilter(Filter) setFilter}. */ -public class Get implements Writable, Row, Comparable { +public class Get extends DatabaseCommand + implements Writable, Row, Comparable { private static final byte GET_VERSION = (byte)2; private byte [] row = null; @@ -352,55 +355,71 @@ } /** - * @return String + * Compile the table and column family (i.e. schema) information + * into a String. Useful for parsing and aggregation by debugging, + * logging, and administration tools. + * @return Map */ @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("row="); - sb.append(Bytes.toStringBinary(this.row)); - sb.append(", maxVersions="); - sb.append("").append(this.maxVersions); - sb.append(", cacheBlocks="); - sb.append(this.cacheBlocks); - sb.append(", timeRange="); - sb.append("[").append(this.tr.getMin()).append(","); - sb.append(this.tr.getMax()).append(")"); - sb.append(", families="); - if(this.familyMap.size() == 0) { - sb.append("ALL"); - return sb.toString(); + public Map getFingerprint() { + Map map = new HashMap(); + List families = new ArrayList(); + map.put("families", families); + for(Map.Entry> entry : + this.familyMap.entrySet()) { + families.add(Bytes.toStringBinary(entry.getKey())); } - boolean moreThanOne = false; + return map; + } + + /** + * Compile the details beyond the scope of getFingerprint (row, columns, + * timestamps, etc.) into a Map along with the fingerprinted information. + * Useful for debugging, logging, and administration tools. + * @param maxCols a limit on the number of columns output prior to truncation + * @return Map + */ + @Override + public Map getDetails(int maxCols) { + // we start with the fingerprint map and build on top of it. + Map map = getFingerprint(); + // replace the fingerprint's simple list of families with a + // map from column families to lists of qualifiers and kv details + Map> columns = new HashMap>(); + map.put("families", columns); + // add scalar information first + map.put("row", Bytes.toStringBinary(this.row)); + map.put("maxVersions", this.maxVersions); + map.put("cacheBlocks", this.cacheBlocks); + List timeRange = new ArrayList(); + timeRange.add(this.tr.getMin()); + timeRange.add(this.tr.getMax()); + map.put("timeRange", timeRange); + int colCount = 0; + // iterate through affected families and add details for(Map.Entry> entry : this.familyMap.entrySet()) { - if(moreThanOne) { - sb.append("), "); - } else { - moreThanOne = true; - sb.append("{"); - } - sb.append("(family="); - sb.append(Bytes.toString(entry.getKey())); - sb.append(", columns="); + List familyList = new ArrayList(); + columns.put(Bytes.toStringBinary(entry.getKey()), familyList); if(entry.getValue() == null) { - sb.append("ALL"); + colCount++; + --maxCols; + familyList.add("ALL"); } else { - sb.append("{"); - boolean moreThanOneB = false; + colCount += entry.getValue().size(); + if (maxCols <= 0) { + continue; + } for(byte [] column : entry.getValue()) { - if(moreThanOneB) { - sb.append(", "); - } else { - moreThanOneB = true; + if (--maxCols <= 0) { + continue; } - sb.append(Bytes.toStringBinary(column)); + familyList.add(Bytes.toStringBinary(column)); } - sb.append("}"); - } - } - sb.append("}"); - return sb.toString(); + } + } + map.put("totalColumns", colCount); + return map; } //Row Index: src/main/java/org/apache/hadoop/hbase/client/DatabaseCommand.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/client/DatabaseCommand.java (revision 0) +++ src/main/java/org/apache/hadoop/hbase/client/DatabaseCommand.java (revision 0) @@ -0,0 +1,101 @@ +/* + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.util.Map; + +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONObject; + +/** + * Superclass for any type that maps to a potentially application-level query. + * (e.g. Put, Get, Delete, Scan, Next, etc.) + * Contains methods for exposure to logging and debugging tools. + */ +public abstract class DatabaseCommand { + // TODO make this configurable + private static final int DEFAULT_MAX_COLS = 5; + + /** + * Produces a String containing a fingerprint which identifies the type and + * the static schema components of a query (i.e. column families) + * @return a map containing fingerprint information (i.e. column families) + */ + public abstract Map getFingerprint(); + + /** + * Produces a String containing a summary of the details of a query + * beyond the scope of the fingerprint (i.e. columns, rows...) + * @param maxCols a limit on the number of columns output prior to truncation + * @return a map containing parameters of a query (i.e. rows, columns...) + */ + public abstract Map getDetails(int maxCols); + + /** + * Produces a full summary of a query + * @return a map containing parameters of a query (i.e. rows, columns...) + */ + public Map getDetails() { + return getDetails(DEFAULT_MAX_COLS); + } + + /** + * Produces a JSON object for fingerprint and details exposure in a + * parseable format. + * @param maxCols a limit on the number of columns to include in the JSON + * @return a JSONObject containing this DatabaseCommand's information + */ + public JSONObject toJSON(int maxCols) { + return new JSONObject(getDetails(maxCols)); + } + + /** + * Produces a JSON object sufficient for description of a query + * in a debugging or logging context. + * @return the produced JSON object + */ + public JSONObject toJSON() { + return toJSON(DEFAULT_MAX_COLS); + } + + /** + * Produces a JSON-parseable string guaranteed to have the full fingerprint, + * but which may have a truncated summary given the suggestedMaxSize + * parameter. + * @param maxCols a limit on the number of columns output in the summary + * prior to truncation + * a max guideline for when to truncate summary output + * @return a JSON-parseable String + */ + public String toString(int maxCols) { + return toJSON(maxCols).toString(); + } + + /** + * Produces a JSON-parseable String sufficient for description of a query + * in a debugging or logging context. + * @return String + */ + @Override + public String toString() { + return toString(DEFAULT_MAX_COLS); + } +} + Index: src/main/java/org/apache/hadoop/hbase/client/Row.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/client/Row.java (revision 1158066) +++ src/main/java/org/apache/hadoop/hbase/client/Row.java (working copy) @@ -21,6 +21,9 @@ import org.apache.hadoop.io.WritableComparable; +import java.util.Map; +import java.util.NavigableSet; + /** * Has a row. */ @@ -29,4 +32,24 @@ * @return The row. */ public byte [] getRow(); + + /** + * @return A map from families to qualifiers affected by this row operation. + */ + public Map> getFamilyMap(); + + /** + * Produces a String containing a fingerprint which identifies the type and + * the static schema components of a query (i.e. column families) + * @return a map containing fingerprint information (i.e. column families) + */ + public abstract Map getFingerprint(); + + /** + * Produces a String containing a summary of the details of a query + * beyond the scope of the fingerprint (i.e. columns, rows...) + * @param maxCols a limit on the number of columns output prior to truncation + * @return a map containing parameters of a query (i.e. rows, columns...) + */ + public abstract Map getDetails(int maxCols); } Index: src/main/java/org/apache/hadoop/hbase/client/Action.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/client/Action.java (revision 1158066) +++ src/main/java/org/apache/hadoop/hbase/client/Action.java (working copy) @@ -28,7 +28,7 @@ import org.apache.hadoop.io.Writable; /* - * A Get, Put or Delete associated with it's region. Used internally by + * A Get, Put or Delete associated with its region. Used internally by * {@link HTable::batch} to associate the action with it's region and maintain * the index from the original request. */ Index: src/main/java/org/apache/hadoop/hbase/client/MultiPut.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/client/MultiPut.java (revision 1158066) +++ src/main/java/org/apache/hadoop/hbase/client/MultiPut.java (working copy) @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.client; +import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HServerAddress; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.Writable; @@ -29,17 +30,23 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; +import java.util.TreeSet; /** * @deprecated Use MultiAction instead * Data type class for putting multiple regions worth of puts in one RPC. */ -public class MultiPut implements Writable { +public class MultiPut extends DatabaseCommand implements Writable { public HServerAddress address; // client code ONLY + public static final int DEFAULT_MAX_PUT_OUTPUT = 10; + // map of regions to lists of puts for that region. public Map > puts = new TreeMap>(Bytes.BYTES_COMPARATOR); @@ -81,8 +88,117 @@ return res; } + /** + * Compile the table and column family (i.e. schema) information + * into a String. Useful for parsing and aggregation by debugging, + * logging, and administration tools. + * @return Map + */ + @Override + public Map getFingerprint() { + Map map = new HashMap(); + // for extensibility, we have a map of table information that we will + // populate with only family information for each table + Map tableInfo = + new HashMap(); + map.put("tables", tableInfo); + for (Map.Entry> entry : puts.entrySet()) { + // our fingerprint only concerns itself with which families are touched, + // not how many Puts touch them, so we use this Set to do just that. + Set familySet; + try { + // since the puts are stored by region, we may have already + // recorded families for this region. if that is the case, + // we want to add to the existing Set. if not, we make a new Set. + String tableName = Bytes.toStringBinary( + HRegionInfo.parseRegionName(entry.getKey())[0]); + if(tableInfo.get(tableName) == null) { + Map table = new HashMap(); + familySet = new TreeSet(); + table.put("families", familySet); + tableInfo.put(tableName, table); + } else { + familySet = (Set) tableInfo.get(tableName).get("families"); + } + } catch (IOException ioe) { + // in the case of parse error, default to labeling by region + Map table = new HashMap(); + familySet = new TreeSet(); + table.put("families", familySet); + tableInfo.put(Bytes.toStringBinary(entry.getKey()), table); + } + // we now iterate through each Put and keep track of which families + // are affected in this table. + for (Put p : entry.getValue()) { + for (byte[] fam : p.getFamilyMap().keySet()) { + familySet.add(Bytes.toStringBinary(fam)); + } + } + } + return map; + } + /** + * Compile the details beyond the scope of getFingerprint (mostly + * getDetails from the Puts) into a Map along with the fingerprinted + * information. Useful for debugging, logging, and administration tools. + * @param maxCols a limit on the number of columns output prior to truncation + * @return Map + */ @Override + public Map getDetails(int maxCols) { + Map map = getFingerprint(); + Map tableInfo = (Map) map.get("tables"); + int putCount = 0; + for (Map.Entry> entry : puts.entrySet()) { + // If the limit has been hit for put output, just adjust our counter + if(putCount >= DEFAULT_MAX_PUT_OUTPUT) { + continue; + } + List regionPuts = entry.getValue(); + List> putSummaries = + new ArrayList>(); + // find out how many of this region's puts we can add without busting + // the maximum + int regionPutsToAdd = regionPuts.size(); + putCount += regionPutsToAdd; + if(putCount > DEFAULT_MAX_PUT_OUTPUT) { + regionPutsToAdd -= putCount - DEFAULT_MAX_PUT_OUTPUT; + } + for (Iterator iter = regionPuts.iterator(); regionPutsToAdd-- > 0;) { + putSummaries.add(iter.next().getDetails(maxCols)); + } + // attempt to extract the table name from the region name + String tableName = ""; + try { + tableName = Bytes.toStringBinary( + HRegionInfo.parseRegionName(entry.getKey())[0]); + } catch (IOException ioe) { + // in the case of parse error, default to labeling by region + tableName = Bytes.toStringBinary(entry.getKey()); + } + // since the puts are stored by region, we may have already + // recorded puts for this region. if that is the case, + // we want to add to the existing List. if not, we place a new list + // in the map + Map table = + (Map) tableInfo.get(tableName); + if(table == null) { + // in case the Put has changed since getFingerprint's map was built + table = new HashMap(); + tableInfo.put(tableName, table); + table.put("puts", putSummaries); + } else if (table.get("puts") == null) { + table.put("puts", putSummaries); + } else { + ((List>) table.get("puts")).addAll(putSummaries); + } + } + map.put("totalPuts", putCount); + return map; + } + + @Override public void write(DataOutput out) throws IOException { out.writeInt(puts.size()); for( Map.Entry> e : puts.entrySet()) { Index: src/main/java/org/apache/hadoop/hbase/client/Scan.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/client/Scan.java (revision 1158066) +++ src/main/java/org/apache/hadoop/hbase/client/Scan.java (working copy) @@ -34,8 +34,10 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.NavigableSet; import java.util.TreeMap; @@ -81,7 +83,7 @@ * Expert: To explicitly disable server-side block caching for this scan, * execute {@link #setCacheBlocks(boolean)}. */ -public class Scan implements Writable { +public class Scan extends DatabaseCommand implements Writable { private static final byte SCAN_VERSION = (byte)2; private byte [] startRow = HConstants.EMPTY_START_ROW; private byte [] stopRow = HConstants.EMPTY_END_ROW; @@ -494,60 +496,78 @@ } /** - * @return String + * Compile the table and column family (i.e. schema) information + * into a String. Useful for parsing and aggregation by debugging, + * logging, and administration tools. + * @return Map */ @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("startRow="); - sb.append(Bytes.toStringBinary(this.startRow)); - sb.append(", stopRow="); - sb.append(Bytes.toStringBinary(this.stopRow)); - sb.append(", maxVersions="); - sb.append(this.maxVersions); - sb.append(", batch="); - sb.append(this.batch); - sb.append(", caching="); - sb.append(this.caching); - sb.append(", cacheBlocks="); - sb.append(this.cacheBlocks); - sb.append(", timeRange="); - sb.append("[").append(this.tr.getMin()).append(","); - sb.append(this.tr.getMax()).append(")"); - sb.append(", families="); + public Map getFingerprint() { + Map map = new HashMap(); + List families = new ArrayList(); if(this.familyMap.size() == 0) { - sb.append("ALL"); - return sb.toString(); + map.put("families", "ALL"); + return map; + } else { + map.put("families", families); } - boolean moreThanOne = false; - for(Map.Entry> entry : this.familyMap.entrySet()) { - if(moreThanOne) { - sb.append("), "); - } else { - moreThanOne = true; - sb.append("{"); - } - sb.append("(family="); - sb.append(Bytes.toStringBinary(entry.getKey())); - sb.append(", columns="); + for(Map.Entry> entry : this.familyMap.entrySet()) { + families.add(Bytes.toStringBinary(entry.getKey())); + } + return map; + } + + /** + * Compile the details beyond the scope of getFingerprint (row, columns, + * timestamps, etc.) into a Map along with the fingerprinted information. + * Useful for debugging, logging, and administration tools. + * @param maxCols a limit on the number of columns output prior to truncation + * @return Map + */ + @Override + public Map getDetails(int maxCols) { + // start with the fingerpring map and build on top of it + Map map = getFingerprint(); + // map from families to column list replaces fingerprint's list of families + Map> familyColumns = + new HashMap>(); + map.put("families", familyColumns); + // add scalar information first + map.put("startRow", Bytes.toStringBinary(this.startRow)); + map.put("stopRow", Bytes.toStringBinary(this.stopRow)); + map.put("maxVersions", this.maxVersions); + map.put("batch", this.batch); + map.put("caching", this.caching); + map.put("cacheBlocks", this.cacheBlocks); + List timeRange = new ArrayList(); + timeRange.add(this.tr.getMin()); + timeRange.add(this.tr.getMax()); + map.put("timeRange", timeRange); + int colCount = 0; + // iterate through affected families and list out up to maxCols columns + for(Map.Entry> entry : + this.familyMap.entrySet()) { + List columns = new ArrayList(); + familyColumns.put(Bytes.toStringBinary(entry.getKey()), columns); if(entry.getValue() == null) { - sb.append("ALL"); + colCount++; + --maxCols; + columns.add("ALL"); } else { - sb.append("{"); - boolean moreThanOneB = false; + colCount += entry.getValue().size(); + if (maxCols <= 0) { + continue; + } for(byte [] column : entry.getValue()) { - if(moreThanOneB) { - sb.append(", "); - } else { - moreThanOneB = true; + if (--maxCols <= 0) { + continue; } - sb.append(Bytes.toStringBinary(column)); + columns.add(Bytes.toStringBinary(column)); } - sb.append("}"); - } - } - sb.append("}"); - return sb.toString(); + } + } + map.put("totalColumns", colCount); + return map; } @SuppressWarnings("unchecked") Index: src/main/java/org/apache/hadoop/hbase/client/MultiAction.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/client/MultiAction.java (revision 1158066) +++ src/main/java/org/apache/hadoop/hbase/client/MultiAction.java (working copy) @@ -22,23 +22,29 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.hbase.io.HbaseObjectWritable; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HServerAddress; import java.io.DataOutput; import java.io.IOException; import java.io.DataInput; +import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.ArrayList; import java.util.Set; import java.util.TreeMap; +import java.util.TreeSet; /** * Container for Actions (i.e. Get, Delete, or Put), which are grouped by * regionName. Intended to be used with HConnectionManager.processBatch() */ -public final class MultiAction implements Writable { +public final class MultiAction extends DatabaseCommand implements Writable { + public static final int DEFAULT_MAX_ACTION_OUTPUT = 10; + // map of regions to lists of puts/gets/deletes for that region. public Map>> actions = new TreeMap>>( @@ -92,7 +98,118 @@ return res; } + /** + * Compile the table and column family (i.e. schema) information + * into a String. Useful for parsing and aggregation by debugging, + * logging, and administration tools. + * @return Map + */ @Override + public Map getFingerprint() { + Map map = new HashMap(); + // for extensibility, we have a map of table information that we will + // populate with only family information for each table + Map tableInfo = + new HashMap(); + map.put("tables", tableInfo); + for (Map.Entry>> entry : actions.entrySet()) { + // our fingerprint only concerns itself with which families are touched, + // not how many Actions touch them, so we use this Set to do just that. + Set familySet; + try { + // since the actions are stored by region, we may have already + // recorded families for this region. if that is the case, + // we want to add to the existing Set. if not, we make a new Set. + String tableName = Bytes.toStringBinary( + HRegionInfo.parseRegionName(entry.getKey())[0]); + if(tableInfo.get(tableName) == null) { + Map table = new HashMap(); + familySet = new TreeSet(); + table.put("families", familySet); + tableInfo.put(tableName, table); + } else { + familySet = (Set) tableInfo.get(tableName).get("families"); + } + } catch (IOException ioe) { + // in the case of parse error, default to labeling by region + Map table = new HashMap(); + familySet = new TreeSet(); + table.put("families", familySet); + tableInfo.put(Bytes.toStringBinary(entry.getKey()), table); + } + // we now iterate through each Action and keep track of which families + // are affected in this table. + for (Action a : entry.getValue()) { + for (byte[] fam : a.getAction().getFamilyMap().keySet()) { + familySet.add(Bytes.toStringBinary(fam)); + } + } + } + return map; + } + + /** + * Compile the details beyond the scope of getFingerprint (mostly + * getDetails from the Actions) into a Map along with the fingerprinted + * information. Useful for debugging, logging, and administration tools. + * @param maxCols a limit on the number of columns output prior to truncation + * @return Map + */ + @Override + public Map getDetails(int maxCols) { + Map map = getFingerprint(); + Map tableInfo = (Map) map.get("tables"); + int actionCount = 0; + for (Map.Entry>> entry : actions.entrySet()) { + // If the limit has been hit for action output, just adjust our counter + if(actionCount >= DEFAULT_MAX_ACTION_OUTPUT) { + continue; + } + List> regionActions = entry.getValue(); + List> actionSummaries = + new ArrayList>(); + // find out how many of this region's actions we can add without busting + // the maximum + int regionActionsToAdd = regionActions.size(); + actionCount += regionActionsToAdd; + if(actionCount > DEFAULT_MAX_ACTION_OUTPUT) { + regionActionsToAdd -= actionCount - DEFAULT_MAX_ACTION_OUTPUT; + } + for (Iterator> iter = regionActions.iterator(); + regionActionsToAdd-- > 0;) { + actionSummaries.add(iter.next().getAction().getDetails(maxCols)); + } + // attempt to extract the table name from the region name + String tableName = ""; + try { + tableName = Bytes.toStringBinary( + HRegionInfo.parseRegionName(entry.getKey())[0]); + } catch (IOException ioe) { + // in the case of parse error, default to labeling by region + tableName = Bytes.toStringBinary(entry.getKey()); + } + // since the actions are stored by region, we may have already + // recorded actions for this region. if that is the case, + // we want to add to the existing List. if not, we place a new list + // in the map + Map table = + (Map) tableInfo.get(tableName); + if(table == null) { + // in case the Action has changed since getFingerprint's map was built + table = new HashMap(); + tableInfo.put(tableName, table); + table.put("actions", actionSummaries); + } else if (table.get("actions") == null) { + table.put("actions", actionSummaries); + } else { + ((List>) table.get("actions")) + .addAll(actionSummaries); + } + } + map.put("totalActions", actionCount); + return map; + } + @Override public void write(DataOutput out) throws IOException { out.writeInt(actions.size()); for (Map.Entry>> e : actions.entrySet()) {