diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java index d057ce3..f199329 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java @@ -648,12 +648,15 @@ public class HRegionInfo implements Comparable { */ @Override public String toString() { - return "{" + HConstants.NAME + " => '" + + // Put encoded name first as that is what we search for most; having it show in logs first + // helps grepping around. + return "{ ENCODED => " + getEncodedName() + "," + + HConstants.NAME + " => '" + this.regionNameStr + "', STARTKEY => '" + Bytes.toStringBinary(this.startKey) + "', ENDKEY => '" + Bytes.toStringBinary(this.endKey) + - "', ENCODED => " + getEncodedName() + "," + + "', " + (isOffline()? " OFFLINE => true,": "") + (isSplit()? " SPLIT => true,": "") + "}"; } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java index ebab799..3aa8b59 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java @@ -91,9 +91,9 @@ public class ClientScanner extends AbstractClientScanner { * @throws IOException */ public ClientScanner(final Configuration conf, final Scan scan, - final byte[] tableName, HConnection connection) throws IOException { + final byte [] tableName, HConnection connection) throws IOException { if (LOG.isDebugEnabled()) { - LOG.debug("Scan table=" + Bytes.toString(tableName) + LOG.debug("New scan at table=" + Bytes.toString(tableName) + ", startRow=" + Bytes.toStringBinary(scan.getStartRow())); } this.scan = scan; @@ -205,7 +205,7 @@ public class ClientScanner extends AbstractClientScanner { if (LOG.isDebugEnabled() && this.currentRegion != null) { // Only worth logging if NOT first region in scan. - LOG.debug("Advancing internal scanner to startKey at '" + + LOG.debug("Next region, startKey='" + Bytes.toStringBinary(localStartKey) + "'"); } try { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java index 35d44da..e59d925 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java @@ -1189,9 +1189,8 @@ public class HConnectionManager { } if (isNewCacheEntry) { if (LOG.isTraceEnabled()) { - LOG.trace("Cached location for " + - location.getRegionInfo().getRegionNameAsString() + - " is " + location.getHostnamePort()); + LOG.trace("Cached location " + location.getHostnamePort() + " for " + + location.getRegionInfo().getRegionNameAsString()); } } else if (isStaleUpdate && !location.equals(oldLocation)) { if (LOG.isTraceEnabled()) { @@ -2378,11 +2377,13 @@ public class HConnectionManager { private List exceptions = new ArrayList(); private List actions = new ArrayList(); private List addresses = new ArrayList(); + private List timestamps = new ArrayList(); - public void add(Exception ex, Row row, HRegionLocation location) { + public void add(Exception ex, Row row, HRegionLocation location, long ts) { exceptions.add(ex); actions.add(row); addresses.add(location.getHostnamePort()); + this.timestamps.add(ts); } public void rethrowIfAny() throws RetriesExhaustedWithDetailsException { @@ -2399,11 +2400,13 @@ public class HConnectionManager { exceptions.clear(); actions.clear(); addresses.clear(); + timestamps.clear(); return result; } private RetriesExhaustedWithDetailsException makeException() { - return new RetriesExhaustedWithDetailsException(exceptions, actions, addresses); + return new RetriesExhaustedWithDetailsException(exceptions, actions, addresses, + timestamps); } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java index bdcf326..690dd4b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java @@ -209,7 +209,7 @@ public class MetaScanner { if (bytes == null) return null; HRegionInfo info = HRegionInfo.parseFromOrNull(bytes); if (LOG.isTraceEnabled()) { - LOG.trace("Current INFO from scan results = " + info); + LOG.trace("hRegionInfo=" + info); } return info; } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java index a1e2285..53d4762 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java @@ -50,10 +50,12 @@ extends RetriesExhaustedException { List exceptions; List actions; List hostnameAndPort; + List timestamps; public RetriesExhaustedWithDetailsException(List exceptions, List actions, - List hostnameAndPort) { + List hostnameAndPort, + final List timestamps) { super("Failed " + exceptions.size() + " action" + pluralize(exceptions) + ": " + getDesc(exceptions, actions, hostnameAndPort)); @@ -61,6 +63,7 @@ extends RetriesExhaustedException { this.exceptions = exceptions; this.actions = actions; this.hostnameAndPort = hostnameAndPort; + this.timestamps = timestamps; } public List getCauses() { @@ -122,6 +125,10 @@ extends RetriesExhaustedException { public String getExhaustiveDescription() { StringWriter errorWriter = new StringWriter(); PrintWriter pw = new PrintWriter(errorWriter); + // Put limit on how many we print out; there could be THOUSANDS of exceptions in result. + // I saw 6k exceptions thrown when a server went down w/ lots of regions with an + // outstanding bulk write. + int min = Math.min(this.exceptions.size(), 10); for (int i = 0; i < this.exceptions.size(); ++i) { Throwable t = this.exceptions.get(i); Row action = this.actions.get(i); @@ -130,6 +137,9 @@ extends RetriesExhaustedException { if (this.exceptions.size() > 1) { pw.append(" #" + i); } + if (this.timestamps != null) { + pw.append(" at " + this.timestamps.get(i)); + } pw.append(" from " + server + " for " + ((action == null) ? "unknown key" : Bytes.toStringBinary(action.getRow()))); if (t != null) { @@ -137,6 +147,10 @@ extends RetriesExhaustedException { t.printStackTrace(pw); } } + if (this.exceptions.size() > min) { + pw.println(); + pw.println("AND " + (this.exceptions.size() - min) + " more exceptions..."); + } pw.flush(); return errorWriter.toString(); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperRegistry.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperRegistry.java index 5f6f45c..e0b5dd2 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperRegistry.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperRegistry.java @@ -49,15 +49,16 @@ class ZooKeeperRegistry implements Registry { @Override public HRegionLocation getMetaRegionLocation() throws IOException { + // TODO: Cache the location a while rather than ask zookeeper every time. ZooKeeperKeepAliveConnection zkw = hci.getKeepAliveZooKeeperWatcher(); try { if (LOG.isTraceEnabled()) { - LOG.trace("Looking up meta region location in ZK," + " connection=" + this); + LOG.trace("Looking up meta region location, " + this.hci.toString()); } ServerName servername = MetaRegionTracker.blockUntilAvailable(zkw, hci.rpcTimeout); if (LOG.isTraceEnabled()) { - LOG.trace("Looked up meta region location, connection=" + this + + LOG.trace("Found meta region location, " + this.hci.toString() + "; serverName=" + ((servername == null) ? "null" : servername)); } if (servername == null) return null; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java index 13b55a1..49110b2 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java @@ -33,9 +33,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.codec.Codec; import org.apache.hadoop.hbase.io.ByteBufferOutputStream; -import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest; -import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest; -import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.compress.CodecPool; import org.apache.hadoop.io.compress.CompressionCodec; @@ -47,7 +44,6 @@ import com.google.common.base.Preconditions; import com.google.protobuf.CodedInputStream; import com.google.protobuf.CodedOutputStream; import com.google.protobuf.Message; -import com.google.protobuf.TextFormat; /** * Utility to help ipc'ing. @@ -88,6 +84,7 @@ class IPCUtil { // TOOD: Reuse buffers? // Presizing doesn't work because can't tell what size will be when serialized. // BBOS will resize itself. + // TODO: Keep stats on how often we get it wrong and adjust. ByteBufferOutputStream baos = new ByteBufferOutputStream(this.cellBlockBuildingInitialBufferSize); OutputStream os = baos;