2011-09-13 01:15:58,063 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10253,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5dc9510f)","client":"172.29.81.118:41784","starttimems":1315901747809,"queuetimems":13652,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,212 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10336,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1d4b476)","client":"172.29.81.117:40675","starttimems":1315901747875,"queuetimems":13479,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,218 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10330,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@51f5c45f)","client":"172.29.81.106:45211","starttimems":1315901747887,"queuetimems":13232,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,230 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10383,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@616676c8)","client":"172.29.81.96:48630","starttimems":1315901747846,"queuetimems":13638,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,245 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10369,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@56f0ae84)","client":"172.29.81.99:44299","starttimems":1315901747875,"queuetimems":13302,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,251 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10374,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@62f1b73e)","client":"172.29.81.111:43060","starttimems":1315901747876,"queuetimems":13261,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,253 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10394,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4e17402d)","client":"172.29.81.105:40413","starttimems":1315901747858,"queuetimems":13473,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,256 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10398,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7979c76)","client":"172.29.81.110:48803","starttimems":1315901747857,"queuetimems":13593,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,269 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10505,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@436e6296)","client":"172.29.81.103:41135","starttimems":1315901747763,"queuetimems":13612,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,274 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10138,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7478832e)","client":"172.29.81.112:53478","starttimems":1315901748135,"queuetimems":13308,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,277 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10149,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@102d9df4)","client":"172.29.81.115:34402","starttimems":1315901748127,"queuetimems":13366,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,289 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10405,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@41e45c9f)","client":"172.29.81.98:57000","starttimems":1315901747883,"queuetimems":13244,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,340 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10136,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7317f743)","client":"172.29.81.119:48906","starttimems":1315901748203,"queuetimems":13256,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,343 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10186,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7110e48f)","client":"172.29.81.114:60702","starttimems":1315901748156,"queuetimems":13286,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,343 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10173,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@c0da247)","client":"172.29.81.100:41763","starttimems":1315901748169,"queuetimems":13265,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,347 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10178,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4ff015c3)","client":"172.29.81.113:48373","starttimems":1315901748168,"queuetimems":13275,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,359 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10153,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@319883b8)","client":"172.29.81.93:60908","starttimems":1315901748204,"queuetimems":13218,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,387 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10173,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5e466819)","client":"172.29.81.94:60973","starttimems":1315901748213,"queuetimems":12897,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,402 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10194,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@f4f3ff5)","client":"172.29.81.95:43027","starttimems":1315901748207,"queuetimems":13195,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,402 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10187,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1beff5ad)","client":"172.29.81.99:44299","starttimems":1315901748214,"queuetimems":12893,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,412 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10249,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1c710d32)","client":"172.29.81.120:57523","starttimems":1315901748162,"queuetimems":13271,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,423 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10209,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@34e024d3)","client":"172.29.81.110:48803","starttimems":1315901748213,"queuetimems":12918,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,427 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10204,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1def4948)","client":"172.29.81.116:34527","starttimems":1315901748222,"queuetimems":12790,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,442 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10235,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@787990c9)","client":"172.29.81.97:45177","starttimems":1315901748206,"queuetimems":13202,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,442 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10233,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2401e55e)","client":"172.29.81.107:48181","starttimems":1315901748207,"queuetimems":13102,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,694 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10452,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@6a847e7f)","client":"172.29.81.118:41784","starttimems":1315901748241,"queuetimems":12775,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,937 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10651,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3c87008a)","client":"172.29.81.101:40653","starttimems":1315901748285,"queuetimems":12812,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,941 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10699,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@209a26d2)","client":"172.29.81.102:60783","starttimems":1315901748241,"queuetimems":12804,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:58,947 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/.tmp/9152928263150119454 to hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/family/3628373290671792133 2011-09-13 01:15:58,965 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10687,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@56378bda)","client":"172.29.81.98:57000","starttimems":1315901748277,"queuetimems":12804,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:59,001 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/family/3628373290671792133, entries=206820, sequenceid=454257, memsize=64.5m, filesize=28.6m 2011-09-13 01:15:59,055 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10836,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@6b3780af)","client":"172.29.81.103:41135","starttimems":1315901748218,"queuetimems":12856,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:15:59,526 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.5m for region usertable,user728323520,1315898727295.a623bd664ea942437bf6ae1073fb5560. in 1335ms, sequenceid=454257, compaction requested=true 2011-09-13 01:15:59,751 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315901735870, entries=3942, filesize=67694024. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315901759526 2011-09-13 01:16:22,693 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/.tmp/6621965761491655627 to hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/family/5256924654000361691 2011-09-13 01:16:22,753 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/family/5256924654000361691, entries=206200, sequenceid=455648, memsize=64.3m, filesize=28.5m 2011-09-13 01:16:22,769 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.3m for region usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd. in 19503ms, sequenceid=455648, compaction requested=true 2011-09-13 01:16:22,910 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315901759526, entries=5275, filesize=89962357. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315901782769 2011-09-13 01:16:22,910 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315901542323 whose highest sequenceid is 435017 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315901542323 2011-09-13 01:16:26,010 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/8c36971138a4bfebe5aeb94c163f2a81/.tmp/4185103976041931651 to hdfs://c0309.xxxxx:17020/hbase/usertable/8c36971138a4bfebe5aeb94c163f2a81/family/4062595437497446409 2011-09-13 01:16:26,046 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/8c36971138a4bfebe5aeb94c163f2a81/family/4062595437497446409, entries=204790, sequenceid=460077, memsize=64.0m, filesize=28.5m 2011-09-13 01:16:26,314 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.0m for region usertable,user1274566160,1315898727282.8c36971138a4bfebe5aeb94c163f2a81. in 1990ms, sequenceid=460077, compaction requested=false 2011-09-13 01:16:30,307 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 4 file(s) in family of usertable,user980023560,1315898727298.a4521b468fb6ac94dd495ee2db515ae0.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/family/1873768012455179962, size=184.6m; total size for store is 496.4m 2011-09-13 01:16:30,307 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user980023560,1315898727298.a4521b468fb6ac94dd495ee2db515ae0., storeName=family, fileCount=4, fileSize=184.6m, priority=2, time=3915972298178297; duration=42sec 2011-09-13 01:16:30,307 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d. 2011-09-13 01:16:30,307 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 4 file(s) in family of usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d. into hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/.tmp, seqid=447385, totalSize=173.5m 2011-09-13 01:16:56,425 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/.tmp/537681719932587744 to hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/family/5498769807365879394 2011-09-13 01:16:56,551 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/family/5498769807365879394, entries=204900, sequenceid=461519, memsize=64.1m, filesize=28.5m 2011-09-13 01:16:56,787 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.1m for region usertable,user1772610920,1315898727287.77b5b2f51884554ebf951146d25f1bd2. in 13488ms, sequenceid=461519, compaction requested=false 2011-09-13 01:16:57,400 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315901782769, entries=5500, filesize=94449750. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315901816787 2011-09-13 01:17:01,930 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/169b91a09b9b10a23602aaea5e8b9fc2/.tmp/5730202307176484776 to hdfs://c0309.xxxxx:17020/hbase/usertable/169b91a09b9b10a23602aaea5e8b9fc2/family/8606524391041101206 2011-09-13 01:17:01,991 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/169b91a09b9b10a23602aaea5e8b9fc2/family/8606524391041101206, entries=222570, sequenceid=465278, memsize=69.6m, filesize=31.0m 2011-09-13 01:17:01,998 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~69.6m for region usertable,user1901138600,1315898727288.169b91a09b9b10a23602aaea5e8b9fc2. in 5210ms, sequenceid=465278, compaction requested=false 2011-09-13 01:17:06,423 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/.tmp/5441254326507002578 to hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/family/2120511048224416284 2011-09-13 01:17:07,041 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 4 file(s) in family of usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/family/4668008532096776496, size=173.5m; total size for store is 500.3m 2011-09-13 01:17:07,041 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d., storeName=family, fileCount=4, fileSize=173.5m, priority=2, time=3915998772391297; duration=36sec 2011-09-13 01:17:07,041 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user728323520,1315898727295.a623bd664ea942437bf6ae1073fb5560. 2011-09-13 01:17:07,041 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 4 file(s) in family of usertable,user728323520,1315898727295.a623bd664ea942437bf6ae1073fb5560. into hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/.tmp, seqid=454257, totalSize=170.8m 2011-09-13 01:17:07,466 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/family/2120511048224416284, entries=207030, sequenceid=465564, memsize=64.7m, filesize=28.8m 2011-09-13 01:17:07,471 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.7m for region usertable,user2029666280,1315898727289.8cb227d0fbbb2ad66e2f3d25cdfa0f89. in 5473ms, sequenceid=465564, compaction requested=false 2011-09-13 01:17:32,010 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/766e5174f1140d766dd0af91cbd1d845/.tmp/2760770737505437141 to hdfs://c0309.xxxxx:17020/hbase/usertable/766e5174f1140d766dd0af91cbd1d845/family/2640966823655511963 2011-09-13 01:17:32,054 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/766e5174f1140d766dd0af91cbd1d845/family/2640966823655511963, entries=205410, sequenceid=467410, memsize=64.1m, filesize=28.4m 2011-09-13 01:17:32,058 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.1m for region usertable,user224923440,1315898727290.766e5174f1140d766dd0af91cbd1d845. in 755ms, sequenceid=467410, compaction requested=false 2011-09-13 01:17:38,254 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 4 file(s) in family of usertable,user728323520,1315898727295.a623bd664ea942437bf6ae1073fb5560.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/family/7289868371714560561, size=170.8m; total size for store is 507.3m 2011-09-13 01:17:38,254 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user728323520,1315898727295.a623bd664ea942437bf6ae1073fb5560., storeName=family, fileCount=4, fileSize=170.8m, priority=2, time=3916060784280297; duration=31sec 2011-09-13 01:17:38,254 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user1397738520,1315898727283.d057127fe98e78ea72dc9be657e68a4a. 2011-09-13 01:17:38,255 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 4 file(s) in family of usertable,user1397738520,1315898727283.d057127fe98e78ea72dc9be657e68a4a. into hdfs://c0309.xxxxx:17020/hbase/usertable/d057127fe98e78ea72dc9be657e68a4a/.tmp, seqid=419737, totalSize=473.8m 2011-09-13 01:18:12,935 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315901816787, entries=3771, filesize=64238709. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315901891048 2011-09-13 01:18:12,938 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315901561984 whose highest sequenceid is 439060 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315901561984 2011-09-13 01:18:12,969 WARN org.apache.hadoop.hdfs.DFSClient: Error while syncing java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1041) 2011-09-13 01:18:12,972 FATAL org.apache.hadoop.hbase.regionserver.wal.HLog: Could not sync. Requesting close of hlog java.io.IOException: Reflection at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:194) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1041) Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) ... 3 more Caused by: java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) ... 7 more 2011-09-13 01:18:12,972 ERROR org.apache.hadoop.hbase.regionserver.wal.HLog: Error while syncing, requesting close of hlog java.io.IOException: Reflection at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:194) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1041) Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) ... 3 more Caused by: java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) ... 7 more 2011-09-13 01:18:12,981 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315901590889 whose highest sequenceid is 442811 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315901590889 2011-09-13 01:18:21,488 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/.tmp/355156118970379463 to hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/family/2490061962006598742 2011-09-13 01:18:21,555 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/family/2490061962006598742, entries=205880, sequenceid=470185, memsize=64.4m, filesize=28.7m 2011-09-13 01:18:21,745 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.4m for region usertable,user1022866120,1315898727279.f3cb8bb14102b43abbb2b45cd2fc4945. in 3240ms, sequenceid=470185, compaction requested=true 2011-09-13 01:18:29,027 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/.tmp/1996798591839286705 to hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/family/1722818928097890964 2011-09-13 01:18:29,088 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/family/1722818928097890964, entries=210520, sequenceid=471104, memsize=65.7m, filesize=29.1m 2011-09-13 01:18:29,338 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~65.7m for region usertable,user980023560,1315898727298.a4521b468fb6ac94dd495ee2db515ae0. in 7593ms, sequenceid=471104, compaction requested=false 2011-09-13 01:18:30,549 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315901891048, entries=4444, filesize=75545524. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315901909338 2011-09-13 01:18:35,999 INFO org.apache.hadoop.hbase.regionserver.Store: Completed major compaction of 4 file(s) in family of usertable,user1397738520,1315898727283.d057127fe98e78ea72dc9be657e68a4a.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/d057127fe98e78ea72dc9be657e68a4a/family/649989683799361124, size=473.8m; total size for store is 505.0m 2011-09-13 01:18:36,000 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user1397738520,1315898727283.d057127fe98e78ea72dc9be657e68a4a., storeName=family, fileCount=4, fileSize=473.8m, priority=3, time=3915783446533297; duration=57sec 2011-09-13 01:18:36,000 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd. 2011-09-13 01:18:36,000 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 4 file(s) in family of usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd. into hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/.tmp, seqid=455648, totalSize=508.6m 2011-09-13 01:18:39,188 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/.tmp/1716145900103437496 to hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/family/8158869502657235273 2011-09-13 01:18:39,293 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/family/8158869502657235273, entries=224950, sequenceid=473574, memsize=70.3m, filesize=31.3m 2011-09-13 01:18:39,297 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~70.3m for region usertable,user1146038480,1315898727281.f6d7a2cd3251b638e0f60e6e46e71a88. in 9959ms, sequenceid=473574, compaction requested=true 2011-09-13 01:18:44,333 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/d057127fe98e78ea72dc9be657e68a4a/.tmp/4802361950165037599 to hdfs://c0309.xxxxx:17020/hbase/usertable/d057127fe98e78ea72dc9be657e68a4a/family/6169214196692403170 2011-09-13 01:18:44,406 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/d057127fe98e78ea72dc9be657e68a4a/family/6169214196692403170, entries=227440, sequenceid=474907, memsize=71.1m, filesize=31.7m 2011-09-13 01:18:44,415 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~71.1m for region usertable,user1397738520,1315898727283.d057127fe98e78ea72dc9be657e68a4a. in 5117ms, sequenceid=474907, compaction requested=false 2011-09-13 01:18:46,561 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/50d20bc0e631f85cd8b961fcf983914a/.tmp/8127224861659706687 to hdfs://c0309.xxxxx:17020/hbase/usertable/50d20bc0e631f85cd8b961fcf983914a/family/7711684226715942973 2011-09-13 01:18:46,596 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/50d20bc0e631f85cd8b961fcf983914a/family/7711684226715942973, entries=229770, sequenceid=474908, memsize=71.9m, filesize=32.0m 2011-09-13 01:18:46,600 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~71.9m for region usertable,user1526266200,1315898727285.50d20bc0e631f85cd8b961fcf983914a. in 2185ms, sequenceid=474908, compaction requested=true 2011-09-13 01:18:47,845 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/.tmp/8637071800325094016 to hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/family/5159190306318961027 2011-09-13 01:18:47,905 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/family/5159190306318961027, entries=218480, sequenceid=474909, memsize=68.3m, filesize=30.4m 2011-09-13 01:18:47,908 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~68.3m for region usertable,user1649438560,1315898727286.29ec66424227dc3cbe3d8b2311764fc9. in 1307ms, sequenceid=474909, compaction requested=true 2011-09-13 01:18:49,371 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/.tmp/3664018432923032661 to hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/family/3104282282120267310 2011-09-13 01:18:49,686 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/family/3104282282120267310, entries=220300, sequenceid=474910, memsize=68.7m, filesize=30.5m 2011-09-13 01:18:49,689 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~68.7m for region usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d. in 1781ms, sequenceid=474910, compaction requested=false 2011-09-13 01:19:01,843 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315901909338, entries=3747, filesize=63855601. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315901941754 2011-09-13 01:19:01,843 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315901647381 whose highest sequenceid is 446714 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315901647381 2011-09-13 01:19:01,867 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315901683371 whose highest sequenceid is 450560 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315901683371 2011-09-13 01:19:31,913 WARN org.apache.hadoop.hbase.regionserver.wal.HLog: IPC Server handler 19 on 60020 took 1262 ms appending an edit to hlog; editcount=2790, len~=15.6k 2011-09-13 01:19:32,237 INFO org.apache.hadoop.hbase.regionserver.Store: Completed major compaction of 4 file(s) in family of usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/family/89056650218797910, size=508.5m; total size for store is 508.5m 2011-09-13 01:19:32,237 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd., storeName=family, fileCount=4, fileSize=508.6m, priority=3, time=3916084026925297; duration=56sec 2011-09-13 01:19:33,294 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10753,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@8f24647)","client":"172.29.81.95:39438","starttimems":1315901962540,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,294 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10535,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@8780cc)","client":"172.29.81.104:47938","starttimems":1315901962758,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,295 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10868,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@59e3e3e7)","client":"172.29.81.96:36480","starttimems":1315901962425,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,304 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10820,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@227002ce)","client":"172.29.81.97:37274","starttimems":1315901962483,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,318 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10732,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@62a99059)","client":"172.29.81.102:60905","starttimems":1315901962585,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,324 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10666,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@8c33d29)","client":"172.29.81.93:33101","starttimems":1315901962657,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,592 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10902,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@241d2ac4)","client":"172.29.81.107:48547","starttimems":1315901962688,"queuetimems":1,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,598 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11091,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1fbdc141)","client":"172.29.81.120:38955","starttimems":1315901962506,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,603 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10534,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7fee8035)","client":"172.29.81.112:37357","starttimems":1315901963068,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,605 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10881,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5b4a193)","client":"172.29.81.101:41038","starttimems":1315901962723,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,609 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10685,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@d498a34)","client":"172.29.81.98:38205","starttimems":1315901962923,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,610 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10965,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@25673a68)","client":"172.29.81.114:42719","starttimems":1315901962644,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,610 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11041,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@571062bc)","client":"172.29.81.104:47938","starttimems":1315901962568,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,610 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10351,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1c2b9507)","client":"172.29.81.115:34586","starttimems":1315901963258,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,610 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10396,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1674ed0)","client":"172.29.81.93:33101","starttimems":1315901963213,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,610 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10840,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@74c2514b)","client":"172.29.81.102:60905","starttimems":1315901962769,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,619 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11007,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@67496e42)","client":"172.29.81.110:38246","starttimems":1315901962611,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,619 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10659,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2b015efb)","client":"172.29.81.96:36480","starttimems":1315901962959,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,619 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10763,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@6702c6cb)","client":"172.29.81.100:42085","starttimems":1315901962855,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,640 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10430,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3058ff26)","client":"172.29.81.99:40069","starttimems":1315901963209,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,645 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11077,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5db6d9b8)","client":"172.29.81.94:34294","starttimems":1315901962567,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,645 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10260,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@f8fef19)","client":"172.29.81.118:42022","starttimems":1315901963384,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,648 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10293,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@339dca3a)","client":"172.29.81.120:38955","starttimems":1315901963354,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,649 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10798,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@aa2a174)","client":"172.29.81.100:42085","starttimems":1315901962850,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,650 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11038,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3b15cc20)","client":"172.29.81.93:33101","starttimems":1315901962610,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,651 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10309,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5224a92b)","client":"172.29.81.94:34294","starttimems":1315901963341,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,651 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11127,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@64de702e)","client":"172.29.81.101:41038","starttimems":1315901962523,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,651 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11100,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3a4a074a)","client":"172.29.81.100:42085","starttimems":1315901962550,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,651 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10295,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@140c672e)","client":"172.29.81.114:42719","starttimems":1315901963355,"queuetimems":1,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,651 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10399,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@47970fbe)","client":"172.29.81.102:60905","starttimems":1315901963251,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,660 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11037,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@478cf953)","client":"172.29.81.114:42719","starttimems":1315901962622,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,671 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11035,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@6e688284)","client":"172.29.81.107:48547","starttimems":1315901962635,"queuetimems":1,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,890 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10629,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@653b1f12)","client":"172.29.81.100:42085","starttimems":1315901963260,"queuetimems":1,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,890 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11365,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@580f2549)","client":"172.29.81.117:41034","starttimems":1315901962524,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,890 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10521,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@c750aaa)","client":"172.29.81.114:42719","starttimems":1315901963368,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,891 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11355,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4a5908c9)","client":"172.29.81.104:47938","starttimems":1315901962535,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,891 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10677,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1b4fea41)","client":"172.29.81.117:41034","starttimems":1315901963213,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,912 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10695,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@36b4e74)","client":"172.29.81.120:38955","starttimems":1315901963216,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,912 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10705,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@162cd333)","client":"172.29.81.118:42022","starttimems":1315901963206,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,912 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11352,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5d3645fc)","client":"172.29.81.114:42719","starttimems":1315901962559,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,913 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10678,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@79eba36c)","client":"172.29.81.103:51120","starttimems":1315901963233,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,913 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11333,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@58ddc4a4)","client":"172.29.81.95:39438","starttimems":1315901962578,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:33,913 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11359,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@47991890)","client":"172.29.81.117:41034","starttimems":1315901962553,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:34,161 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10905,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4eb44b5d)","client":"172.29.81.120:38955","starttimems":1315901963255,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:34,161 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11210,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@40f9ace8)","client":"172.29.81.98:38205","starttimems":1315901962949,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:34,161 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11529,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@47c6d4ab)","client":"172.29.81.102:60905","starttimems":1315901962631,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:34,162 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11141,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@24291a1e)","client":"172.29.81.96:36480","starttimems":1315901963019,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:34,162 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10980,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4ae647a4)","client":"172.29.81.95:39438","starttimems":1315901963180,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:34,162 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11672,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1ebfef09)","client":"172.29.81.104:47938","starttimems":1315901962488,"queuetimems":1,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:19:34,162 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11557,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5bf6bdd5)","client":"172.29.81.95:39438","starttimems":1315901962603,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} -- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:348) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1226) Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) ... 11 more Caused by: java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) ... 15 more 2011-09-13 01:40:57,670 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315903206527, entries=3769, filesize=64533725. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315903257411 2011-09-13 01:40:57,671 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315903019596 whose highest sequenceid is 591694 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315903019596 2011-09-13 01:41:07,644 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/.tmp/128497747279046309 to hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/family/6260155584735382629 2011-09-13 01:41:07,780 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/family/6260155584735382629, entries=205580, sequenceid=620649, memsize=64.3m, filesize=28.6m 2011-09-13 01:41:10,346 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.3m for region usertable,user2029666280,1315898727289.8cb227d0fbbb2ad66e2f3d25cdfa0f89. in 5792ms, sequenceid=620649, compaction requested=false 2011-09-13 01:41:13,076 WARN org.apache.hadoop.hdfs.DFSClient: Error while syncing java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1041) 2011-09-13 01:41:13,077 WARN org.apache.hadoop.hdfs.DFSClient: Error while syncing java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:1016) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1757) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1627) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2910) at sun.reflect.GeneratedMethodAccessor25.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:348) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1226) 2011-09-13 01:41:13,077 FATAL org.apache.hadoop.hbase.regionserver.wal.HLog: Could not sync. Requesting close of hlog java.io.IOException: Reflection at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:194) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1041) Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) ... 3 more Caused by: java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) ... 7 more 2011-09-13 01:41:13,078 FATAL org.apache.hadoop.hbase.regionserver.wal.HLog: Could not sync. Requesting close of hlog java.io.IOException: Reflection at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:194) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:1016) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1757) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1627) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2910) at sun.reflect.GeneratedMethodAccessor25.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:348) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1226) Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) ... 11 more Caused by: java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) ... 15 more 2011-09-13 01:41:13,078 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315903257411, entries=3745, filesize=63849157. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315903271080 2011-09-13 01:41:13,078 ERROR org.apache.hadoop.hbase.regionserver.wal.HLog: Error while syncing, requesting close of hlog java.io.IOException: Reflection at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:194) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1041) Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) ... 3 more Caused by: java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) ... 7 more 2011-09-13 01:41:13,080 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315903077014 whose highest sequenceid is 595764 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315903077014 2011-09-13 01:41:16,407 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/766e5174f1140d766dd0af91cbd1d845/.tmp/4809340029243467682 to hdfs://c0309.xxxxx:17020/hbase/usertable/766e5174f1140d766dd0af91cbd1d845/family/1071837997330024863 2011-09-13 01:41:16,455 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/766e5174f1140d766dd0af91cbd1d845/family/1071837997330024863, entries=205990, sequenceid=622679, memsize=64.3m, filesize=28.5m 2011-09-13 01:41:18,015 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.3m for region usertable,user224923440,1315898727290.766e5174f1140d766dd0af91cbd1d845. in 4465ms, sequenceid=622679, compaction requested=false 2011-09-13 01:41:20,108 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315903271080, entries=4365, filesize=74465947. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315903278016 2011-09-13 01:41:24,259 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/.tmp/603894261969427593 to hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/family/1996165178783304121 2011-09-13 01:41:24,305 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/family/1996165178783304121, entries=221390, sequenceid=626221, memsize=69.2m, filesize=30.8m 2011-09-13 01:41:25,420 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~69.2m for region usertable,user1022866120,1315898727279.f3cb8bb14102b43abbb2b45cd2fc4945. in 7404ms, sequenceid=626221, compaction requested=true 2011-09-13 01:41:29,380 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/.tmp/276345568948499024 to hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/family/469852105884017483 2011-09-13 01:41:29,418 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/family/469852105884017483, entries=205730, sequenceid=627989, memsize=64.3m, filesize=28.7m 2011-09-13 01:41:29,565 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.3m for region usertable,user1146038480,1315898727281.f6d7a2cd3251b638e0f60e6e46e71a88. in 1696ms, sequenceid=627989, compaction requested=true 2011-09-13 01:41:39,003 WARN org.apache.hadoop.hbase.regionserver.wal.HLog: IPC Server handler 27 on 60020 took 1571 ms appending an edit to hlog; editcount=3827, len~=17.1k 2011-09-13 01:41:39,217 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/d057127fe98e78ea72dc9be657e68a4a/.tmp/1352948411959191767 to hdfs://c0309.xxxxx:17020/hbase/usertable/d057127fe98e78ea72dc9be657e68a4a/family/6527356429643166847 2011-09-13 01:41:39,275 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/d057127fe98e78ea72dc9be657e68a4a/family/6527356429643166847, entries=205510, sequenceid=629369, memsize=64.3m, filesize=28.6m 2011-09-13 01:41:40,752 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.3m for region usertable,user1397738520,1315898727283.d057127fe98e78ea72dc9be657e68a4a. in 6150ms, sequenceid=629369, compaction requested=true 2011-09-13 01:41:42,665 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315903278016, entries=3988, filesize=67452576. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315903300753 2011-09-13 01:41:42,666 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315903089130 whose highest sequenceid is 602524 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315903089130 2011-09-13 01:41:42,669 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10123,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2218e2aa)","client":"172.29.81.120:36969","starttimems":1315903292545,"queuetimems":3307,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:42,670 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10255,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5b6013ad)","client":"172.29.81.100:37528","starttimems":1315903292414,"queuetimems":3183,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:42,676 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10262,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@abf1b9)","client":"172.29.81.101:53673","starttimems":1315903292413,"queuetimems":3184,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:42,678 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10062,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@188aa907)","client":"172.29.81.119:47269","starttimems":1315903292615,"queuetimems":3342,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:42,679 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10063,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@375f1727)","client":"172.29.81.100:37528","starttimems":1315903292615,"queuetimems":3341,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,177 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10226,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@64016598)","client":"172.29.81.108:39785","starttimems":1315903292950,"queuetimems":2432,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,178 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10570,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@17b51173)","client":"172.29.81.107:60374","starttimems":1315903292606,"queuetimems":3338,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,454 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10450,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1132c495)","client":"172.29.81.96:46041","starttimems":1315903293004,"queuetimems":2252,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,479 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10565,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@198a41c)","client":"172.29.81.101:53673","starttimems":1315903292912,"queuetimems":2663,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,485 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10650,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4b43a8f3)","client":"172.29.81.94:47365","starttimems":1315903292834,"queuetimems":3534,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,485 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10529,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4709101f)","client":"172.29.81.107:60374","starttimems":1315903292955,"queuetimems":2420,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,492 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10604,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4894e8a)","client":"172.29.81.96:46041","starttimems":1315903292887,"queuetimems":3423,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,502 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10497,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@27b33cde)","client":"172.29.81.115:60851","starttimems":1315903293004,"queuetimems":2162,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,502 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10536,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@6861272e)","client":"172.29.81.94:47365","starttimems":1315903292965,"queuetimems":2421,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,523 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10623,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2b7bfc62)","client":"172.29.81.96:46041","starttimems":1315903292899,"queuetimems":3416,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,527 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10501,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@399a13e8)","client":"172.29.81.115:60851","starttimems":1315903293025,"queuetimems":1827,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,528 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10550,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4e12f804)","client":"172.29.81.119:47269","starttimems":1315903292977,"queuetimems":2397,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,528 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10628,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3f58f6a7)","client":"172.29.81.106:43767","starttimems":1315903292899,"queuetimems":2660,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,536 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10544,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4f1e56b5)","client":"172.29.81.95:60417","starttimems":1315903292991,"queuetimems":2380,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,539 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10638,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@b1abf07)","client":"172.29.81.96:46041","starttimems":1315903292899,"queuetimems":3400,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,548 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10570,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4fcfa2f7)","client":"172.29.81.99:53711","starttimems":1315903292977,"queuetimems":2401,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,548 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10483,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@48d42f10)","client":"172.29.81.99:53711","starttimems":1315903293064,"queuetimems":1447,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,580 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10586,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@6cbba0e9)","client":"172.29.81.101:53673","starttimems":1315903292964,"queuetimems":2424,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,581 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10740,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@68478327)","client":"172.29.81.108:39785","starttimems":1315903292840,"queuetimems":3464,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,584 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10661,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@36483edc)","client":"172.29.81.95:60417","starttimems":1315903292921,"queuetimems":2655,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,584 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10711,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4116f95b)","client":"172.29.81.108:39785","starttimems":1315903292872,"queuetimems":3484,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,585 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10629,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2d3e23e5)","client":"172.29.81.110:52881","starttimems":1315903292955,"queuetimems":2435,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,586 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10559,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@30329cb)","client":"172.29.81.120:36969","starttimems":1315903293026,"queuetimems":1457,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,586 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10750,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4aa6a697)","client":"172.29.81.94:47365","starttimems":1315903292834,"queuetimems":3495,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,587 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10530,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2d89c601)","client":"172.29.81.95:60417","starttimems":1315903293055,"queuetimems":1441,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,587 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10705,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@30268302)","client":"172.29.81.120:36969","starttimems":1315903292881,"queuetimems":3488,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,599 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10325,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@24ac413d)","client":"172.29.81.94:47365","starttimems":1315903293273,"queuetimems":1558,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,599 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10572,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@28ef1e9a)","client":"172.29.81.101:53673","starttimems":1315903293026,"queuetimems":1467,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,601 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10327,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@63aa3a64)","client":"172.29.81.96:46041","starttimems":1315903293273,"queuetimems":1560,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,603 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10556,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@12fb5ef)","client":"172.29.81.99:53711","starttimems":1315903293046,"queuetimems":1470,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,619 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10356,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5e822462)","client":"172.29.81.99:53711","starttimems":1315903293262,"queuetimems":1594,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,639 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10341,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2494622e)","client":"172.29.81.96:46041","starttimems":1315903293297,"queuetimems":1534,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:43,640 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10538,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3e43e82a)","client":"172.29.81.96:46041","starttimems":1315903293101,"queuetimems":1464,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 01:41:44,882 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/.tmp/3843328827240652821 to hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/family/8735815725916750270 2011-09-13 01:41:44,919 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/family/8735815725916750270, entries=207780, sequenceid=630308, memsize=65.0m, filesize=28.9m 2011-09-13 01:41:44,923 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~65.0m for region usertable,user1649438560,1315898727286.29ec66424227dc3cbe3d8b2311764fc9. in 4170ms, sequenceid=630308, compaction requested=true 2011-09-13 01:41:45,508 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/.tmp/1295250105257294028 to hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/family/8642138129608269065 2011-09-13 01:41:45,539 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/family/8642138129608269065, entries=212200, sequenceid=631085, memsize=66.2m, filesize=29.4m 2011-09-13 01:41:45,548 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~66.2m for region usertable,user980023560,1315898727298.a4521b468fb6ac94dd495ee2db515ae0. in 625ms, sequenceid=631085, compaction requested=false 2011-09-13 01:42:32,298 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315903300753, entries=3812, filesize=64617002. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315903352132 2011-09-13 01:42:32,298 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315903119530 whose highest sequenceid is 606531 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315903119530 2011-09-13 01:42:41,141 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/50d20bc0e631f85cd8b961fcf983914a/.tmp/8139282754735780131 to hdfs://c0309.xxxxx:17020/hbase/usertable/50d20bc0e631f85cd8b961fcf983914a/family/4991994731699278101 2011-09-13 01:42:41,209 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/50d20bc0e631f85cd8b961fcf983914a/family/4991994731699278101, entries=205850, sequenceid=634841, memsize=64.4m, filesize=28.7m 2011-09-13 01:42:41,321 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.4m for region usertable,user1526266200,1315898727285.50d20bc0e631f85cd8b961fcf983914a. in 5891ms, sequenceid=634841, compaction requested=true 2011-09-13 01:42:44,924 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315903352132, entries=3805, filesize=65267137. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315903364515 2011-09-13 01:42:44,924 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315903166871 whose highest sequenceid is 610423 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315903166871 2011-09-13 01:42:46,810 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/.tmp/5124788080370110328 to hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/family/132304403762329863 2011-09-13 01:42:46,873 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/family/132304403762329863, entries=210280, sequenceid=637899, memsize=65.6m, filesize=29.1m 2011-09-13 01:42:46,999 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~65.6m for region usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d. in 2448ms, sequenceid=637899, compaction requested=true 2011-09-13 01:42:46,999 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d. 2011-09-13 01:42:47,000 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 3 file(s) in family of usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d. into hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/.tmp, seqid=637899, totalSize=87.0m 2011-09-13 01:42:52,768 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/.tmp/2810494602542013351 to hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/family/1040486168705684388 2011-09-13 01:42:52,836 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/family/1040486168705684388, entries=207790, sequenceid=639211, memsize=64.8m, filesize=28.7m 2011-09-13 01:42:52,841 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.8m for region usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd. in 4837ms, sequenceid=639211, compaction requested=false 2011-09-13 01:42:54,215 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 3 file(s) in family of usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/family/1243215195463998707, size=87.0m; total size for store is 712.0m 2011-09-13 01:42:54,215 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d., storeName=family, fileCount=3, fileSize=87.0m, priority=3, time=3917668256688297; duration=7sec 2011-09-13 01:43:04,680 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/8c36971138a4bfebe5aeb94c163f2a81/.tmp/2210713532381955865 to hdfs://c0309.xxxxx:17020/hbase/usertable/8c36971138a4bfebe5aeb94c163f2a81/family/8980545988384812061 2011-09-13 01:43:04,737 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/8c36971138a4bfebe5aeb94c163f2a81/family/8980545988384812061, entries=211590, sequenceid=639938, memsize=66.2m, filesize=29.5m 2011-09-13 01:43:05,164 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~66.2m for region usertable,user1274566160,1315898727282.8c36971138a4bfebe5aeb94c163f2a81. in 3653ms, sequenceid=639938, compaction requested=true 2011-09-13 01:43:10,016 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/.tmp/6395926481990907037 to hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/family/5819611466586885410 2011-09-13 01:43:10,094 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/family/5819611466586885410, entries=212160, sequenceid=641215, memsize=66.3m, filesize=29.6m -- 2011-09-13 01:55:48,444 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/d057127fe98e78ea72dc9be657e68a4a/family/6816367016285906892, entries=211970, sequenceid=759463, memsize=66.3m, filesize=29.5m 2011-09-13 01:55:48,446 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~66.3m for region usertable,user1397738520,1315898727283.d057127fe98e78ea72dc9be657e68a4a. in 2157ms, sequenceid=759463, compaction requested=false 2011-09-13 01:55:58,376 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904141405, entries=3952, filesize=67059262. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904156578 2011-09-13 01:55:58,377 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315903943429 whose highest sequenceid is 737327 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315903943429 2011-09-13 01:55:59,560 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 3 file(s) in family of usertable,user224923440,1315898727290.766e5174f1140d766dd0af91cbd1d845.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/766e5174f1140d766dd0af91cbd1d845/family/5034633225472481013, size=88.9m; total size for store is 840.4m 2011-09-13 01:55:59,561 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user224923440,1315898727290.766e5174f1140d766dd0af91cbd1d845., storeName=family, fileCount=3, fileSize=88.9m, priority=2, time=3918417312484297; duration=14sec 2011-09-13 01:56:09,302 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/.tmp/6619123132248769560 to hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/family/4382674364271543285 2011-09-13 01:56:09,333 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/family/4382674364271543285, entries=204800, sequenceid=762641, memsize=64.0m, filesize=28.5m 2011-09-13 01:56:09,623 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.0m for region usertable,user1649438560,1315898727286.29ec66424227dc3cbe3d8b2311764fc9. in 2465ms, sequenceid=762641, compaction requested=true 2011-09-13 01:56:14,262 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/.tmp/8987447894742088143 to hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/family/5594125071838921604 2011-09-13 01:56:14,308 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/family/5594125071838921604, entries=206650, sequenceid=762996, memsize=64.5m, filesize=28.6m 2011-09-13 01:56:14,316 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.5m for region usertable,user980023560,1315898727298.a4521b468fb6ac94dd495ee2db515ae0. in 4692ms, sequenceid=762996, compaction requested=true 2011-09-13 01:56:14,317 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user980023560,1315898727298.a4521b468fb6ac94dd495ee2db515ae0. 2011-09-13 01:56:14,317 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 3 file(s) in family of usertable,user980023560,1315898727298.a4521b468fb6ac94dd495ee2db515ae0. into hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/.tmp, seqid=762996, totalSize=90.3m 2011-09-13 01:56:22,650 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 3 file(s) in family of usertable,user980023560,1315898727298.a4521b468fb6ac94dd495ee2db515ae0.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/family/2967861965031451657, size=90.3m; total size for store is 852.2m 2011-09-13 01:56:22,651 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user980023560,1315898727298.a4521b468fb6ac94dd495ee2db515ae0., storeName=family, fileCount=3, fileSize=90.3m, priority=2, time=3918475573998297; duration=8sec 2011-09-13 01:56:26,504 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/50d20bc0e631f85cd8b961fcf983914a/.tmp/3697505460396751680 to hdfs://c0309.xxxxx:17020/hbase/usertable/50d20bc0e631f85cd8b961fcf983914a/family/3072902507490732568 2011-09-13 01:56:26,536 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/50d20bc0e631f85cd8b961fcf983914a/family/3072902507490732568, entries=204830, sequenceid=763627, memsize=64.1m, filesize=28.5m 2011-09-13 01:56:26,537 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.1m for region usertable,user1526266200,1315898727285.50d20bc0e631f85cd8b961fcf983914a. in 1212ms, sequenceid=763627, compaction requested=true 2011-09-13 01:56:35,194 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904156578, entries=3773, filesize=63929981. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904195008 2011-09-13 01:56:35,195 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315903977590 whose highest sequenceid is 741114 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315903977590 2011-09-13 01:56:40,579 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/.tmp/2372908028087422018 to hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/family/5233690861969810506 2011-09-13 01:56:40,638 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/family/5233690861969810506, entries=207210, sequenceid=766533, memsize=64.7m, filesize=28.7m 2011-09-13 01:56:40,644 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.7m for region usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d. in 1866ms, sequenceid=766533, compaction requested=true 2011-09-13 01:56:46,140 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/.tmp/2407267204576785673 to hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/family/3446700972817900364 2011-09-13 01:56:46,200 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/family/3446700972817900364, entries=205320, sequenceid=767577, memsize=64.1m, filesize=28.4m 2011-09-13 01:56:46,236 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.1m for region usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd. in 741ms, sequenceid=767577, compaction requested=true 2011-09-13 01:56:46,236 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd. 2011-09-13 01:56:46,237 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 5 file(s) in family of usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd. into hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/.tmp, seqid=767577, totalSize=348.5m 2011-09-13 01:56:52,429 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904195008, entries=3777, filesize=64125117. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904212358 2011-09-13 01:56:52,429 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904026543 whose highest sequenceid is 744882 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315904026543 2011-09-13 01:57:05,396 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/8c36971138a4bfebe5aeb94c163f2a81/.tmp/1646332208989655449 to hdfs://c0309.xxxxx:17020/hbase/usertable/8c36971138a4bfebe5aeb94c163f2a81/family/8015345105163023793 2011-09-13 01:57:05,475 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/8c36971138a4bfebe5aeb94c163f2a81/family/8015345105163023793, entries=204860, sequenceid=771381, memsize=64.1m, filesize=28.5m 2011-09-13 01:57:05,692 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.1m for region usertable,user1274566160,1315898727282.8c36971138a4bfebe5aeb94c163f2a81. in 2642ms, sequenceid=771381, compaction requested=true 2011-09-13 01:57:13,646 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904212358, entries=4014, filesize=68693486. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904232334 2011-09-13 01:57:13,646 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904037484 whose highest sequenceid is 748902 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315904037484 2011-09-13 01:57:19,167 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/.tmp/8668445282225411019 to hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/family/1663176309081272375 2011-09-13 01:57:19,292 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/family/1663176309081272375, entries=205210, sequenceid=774413, memsize=64.2m, filesize=28.6m 2011-09-13 01:57:19,298 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.2m for region usertable,user1772610920,1315898727287.77b5b2f51884554ebf951146d25f1bd2. in 2730ms, sequenceid=774413, compaction requested=true 2011-09-13 01:57:39,740 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/169b91a09b9b10a23602aaea5e8b9fc2/.tmp/6635844626695950401 to hdfs://c0309.xxxxx:17020/hbase/usertable/169b91a09b9b10a23602aaea5e8b9fc2/family/3979167821037302387 2011-09-13 01:57:39,789 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/169b91a09b9b10a23602aaea5e8b9fc2/family/3979167821037302387, entries=206170, sequenceid=775946, memsize=64.5m, filesize=28.7m 2011-09-13 01:57:39,796 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.5m for region usertable,user1901138600,1315898727288.169b91a09b9b10a23602aaea5e8b9fc2. in 9061ms, sequenceid=775946, compaction requested=true 2011-09-13 01:57:50,941 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 5 file(s) in family of usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/family/5670546917365810713, size=348.5m; total size for store is 857.1m 2011-09-13 01:57:50,941 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd., storeName=family, fileCount=5, fileSize=348.5m, priority=1, time=3918507493748297; duration=1mins, 4sec 2011-09-13 01:58:07,911 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904232334, entries=3806, filesize=64182336. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904287828 2011-09-13 01:58:18,420 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/.tmp/7828943434575173742 to hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/family/6338536749140302047 2011-09-13 01:58:18,508 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/family/6338536749140302047, entries=205020, sequenceid=777238, memsize=64.1m, filesize=28.6m 2011-09-13 01:58:19,007 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.1m for region usertable,user2029666280,1315898727289.8cb227d0fbbb2ad66e2f3d25cdfa0f89. in 9748ms, sequenceid=777238, compaction requested=true 2011-09-13 01:58:20,519 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/.tmp/3717941242788954895 to hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/family/2797213318665880335 2011-09-13 01:58:20,625 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/family/2797213318665880335, entries=214720, sequenceid=778993, memsize=67.0m, filesize=29.7m 2011-09-13 01:58:20,721 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~67.0m for region usertable,user728323520,1315898727295.a623bd664ea942437bf6ae1073fb5560. in 1713ms, sequenceid=778993, compaction requested=true 2011-09-13 01:58:22,610 WARN org.apache.hadoop.hbase.regionserver.wal.HLog: IPC Server handler 19 on 60020 took 1333 ms appending an edit to hlog; editcount=3348, len~=17.1k 2011-09-13 01:58:23,352 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/766e5174f1140d766dd0af91cbd1d845/.tmp/6794794448096645164 to hdfs://c0309.xxxxx:17020/hbase/usertable/766e5174f1140d766dd0af91cbd1d845/family/3528576821356286760 2011-09-13 01:58:23,426 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/766e5174f1140d766dd0af91cbd1d845/family/3528576821356286760, entries=220650, sequenceid=779609, memsize=68.9m, filesize=30.5m 2011-09-13 01:58:23,761 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~68.9m for region usertable,user224923440,1315898727290.766e5174f1140d766dd0af91cbd1d845. in 2997ms, sequenceid=779609, compaction requested=true 2011-09-13 01:58:24,910 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/.tmp/6390397195037145312 to hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/family/711212040994548514 2011-09-13 01:58:25,126 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/family/711212040994548514, entries=210920, sequenceid=780260, memsize=66.0m, filesize=29.4m 2011-09-13 01:58:25,213 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~66.0m for region usertable,user1022866120,1315898727279.f3cb8bb14102b43abbb2b45cd2fc4945. in 1452ms, sequenceid=780260, compaction requested=true 2011-09-13 01:58:25,367 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904287828, entries=5255, filesize=89528321. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904305213 2011-09-13 01:58:25,367 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904052936 whose highest sequenceid is 753444 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315904052936 2011-09-13 01:58:25,416 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904116055 whose highest sequenceid is 757435 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315904116055 2011-09-13 01:58:50,977 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/.tmp/1062341236959620165 to hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/family/141522885033749202 2011-09-13 01:58:51,023 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/family/141522885033749202, entries=209660, sequenceid=783755, memsize=65.6m, filesize=29.2m 2011-09-13 01:58:51,056 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~65.6m for region usertable,user1146038480,1315898727281.f6d7a2cd3251b638e0f60e6e46e71a88. in 5210ms, sequenceid=783755, compaction requested=true 2011-09-13 01:58:51,057 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user1146038480,1315898727281.f6d7a2cd3251b638e0f60e6e46e71a88. 2011-09-13 01:58:51,057 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 4 file(s) in family of usertable,user1146038480,1315898727281.f6d7a2cd3251b638e0f60e6e46e71a88. into hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/.tmp, seqid=783755, totalSize=322.0m 2011-09-13 01:59:00,552 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/d057127fe98e78ea72dc9be657e68a4a/.tmp/7530704983931368372 to hdfs://c0309.xxxxx:17020/hbase/usertable/d057127fe98e78ea72dc9be657e68a4a/family/5143389040676295509 2011-09-13 01:59:00,609 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/d057127fe98e78ea72dc9be657e68a4a/family/5143389040676295509, entries=206240, sequenceid=785289, memsize=64.5m, filesize=28.7m 2011-09-13 01:59:01,229 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.5m for region usertable,user1397738520,1315898727283.d057127fe98e78ea72dc9be657e68a4a. in 6286ms, sequenceid=785289, compaction requested=true 2011-09-13 01:59:01,692 WARN org.apache.hadoop.hdfs.DFSClient: Error while syncing java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1041) 2011-09-13 01:59:01,692 FATAL org.apache.hadoop.hbase.regionserver.wal.HLog: Could not sync. Requesting close of hlog java.io.IOException: Reflection at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:194) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1041) Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) ... 3 more Caused by: java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) ... 7 more 2011-09-13 01:59:01,693 ERROR org.apache.hadoop.hbase.regionserver.wal.HLog: Error while syncing, requesting close of hlog java.io.IOException: Reflection at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:194) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1041) Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) ... 3 more Caused by: java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) ... 7 more 2011-09-13 01:59:01,694 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904305213, entries=4383, filesize=74936003. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904341229 2011-09-13 01:59:01,695 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904141405 whose highest sequenceid is 761387 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315904141405 2011-09-13 01:59:06,132 WARN org.apache.hadoop.hbase.regionserver.wal.HLog: IPC Server handler 96 on 60020 took 1063 ms appending an edit to hlog; editcount=723, len~=14.2k 2011-09-13 01:59:12,974 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/.tmp/515503910901206215 to hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/family/772192888054987902 2011-09-13 01:59:13,052 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/family/772192888054987902, entries=208160, sequenceid=788265, memsize=65.1m, filesize=29.0m 2011-09-13 01:59:13,055 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~65.1m for region usertable,user1649438560,1315898727286.29ec66424227dc3cbe3d8b2311764fc9. in 6228ms, sequenceid=788265, compaction requested=true 2011-09-13 01:59:13,104 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904341229, entries=5400, filesize=92122842. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904353055 2011-09-13 01:59:19,714 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/50d20bc0e631f85cd8b961fcf983914a/.tmp/3932873186904854154 to hdfs://c0309.xxxxx:17020/hbase/usertable/50d20bc0e631f85cd8b961fcf983914a/family/1619996198911758570 2011-09-13 01:59:19,834 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/50d20bc0e631f85cd8b961fcf983914a/family/1619996198911758570, entries=228260, sequenceid=791796, memsize=71.4m, filesize=31.8m 2011-09-13 01:59:19,840 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~71.4m for region usertable,user1526266200,1315898727285.50d20bc0e631f85cd8b961fcf983914a. in 6785ms, sequenceid=791796, compaction requested=true 2011-09-13 01:59:26,467 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/.tmp/4263539526250472261 to hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/family/4934307306344080360 2011-09-13 01:59:26,521 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/family/4934307306344080360, entries=240820, sequenceid=792507, memsize=75.1m, filesize=33.3m 2011-09-13 01:59:26,749 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~75.1m for region usertable,user980023560,1315898727298.a4521b468fb6ac94dd495ee2db515ae0. in 6909ms, sequenceid=792507, compaction requested=true 2011-09-13 01:59:38,502 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/.tmp/7065524106811191388 to hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/family/5525692812321254845 2011-09-13 01:59:38,633 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/family/5525692812321254845, entries=211950, sequenceid=792612, memsize=66.1m, filesize=29.3m 2011-09-13 01:59:38,637 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~66.1m for region usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d. in 11888ms, sequenceid=792612, compaction requested=true 2011-09-13 01:59:38,637 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d. 2011-09-13 01:59:38,637 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 3 file(s) in family of usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d. into hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/.tmp, seqid=792612, totalSize=86.5m 2011-09-13 01:59:47,298 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 4 file(s) in family of usertable,user1146038480,1315898727281.f6d7a2cd3251b638e0f60e6e46e71a88.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/family/2317938533799814341, size=322.0m; total size for store is 886.0m 2011-09-13 01:59:47,298 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user1146038480,1315898727281.f6d7a2cd3251b638e0f60e6e46e71a88., storeName=family, fileCount=4, fileSize=322.0m, priority=2, time=3918632314260297; duration=56sec 2011-09-13 01:59:47,299 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user1649438560,1315898727286.29ec66424227dc3cbe3d8b2311764fc9. 2011-09-13 01:59:47,299 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 4 file(s) in family of usertable,user1649438560,1315898727286.29ec66424227dc3cbe3d8b2311764fc9. into hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/.tmp, seqid=788265, totalSize=325.2m 2011-09-13 01:59:48,715 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 3 file(s) in family of usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/family/8775017151122062811, size=86.5m; total size for store is 885.6m 2011-09-13 01:59:48,715 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d., storeName=family, fileCount=3, fileSize=86.5m, priority=2, time=3918679894454297; duration=10sec 2011-09-13 02:00:05,601 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/.tmp/4199720036538685405 to hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/family/2413587328297660389 2011-09-13 02:00:05,642 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/family/2413587328297660389, entries=205330, sequenceid=792900, memsize=64.1m, filesize=28.4m 2011-09-13 02:00:05,649 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.1m for region usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd. in 7850ms, sequenceid=792900, compaction requested=false 2011-09-13 02:00:16,725 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904353055, entries=3722, filesize=63938838. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904416647 2011-09-13 02:00:16,725 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904156578 whose highest sequenceid is 765160 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315904156578 2011-09-13 02:00:16,740 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904195008 whose highest sequenceid is 768937 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315904195008 2011-09-13 02:00:21,524 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/8c36971138a4bfebe5aeb94c163f2a81/.tmp/8084365550587877462 to hdfs://c0309.xxxxx:17020/hbase/usertable/8c36971138a4bfebe5aeb94c163f2a81/family/8646812165045474379 2011-09-13 02:00:21,558 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/8c36971138a4bfebe5aeb94c163f2a81/family/8646812165045474379, entries=204930, sequenceid=796569, memsize=64.1m, filesize=28.5m 2011-09-13 02:00:21,559 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.1m for region usertable,user1274566160,1315898727282.8c36971138a4bfebe5aeb94c163f2a81. in 2279ms, sequenceid=796569, compaction requested=true 2011-09-13 02:00:28,941 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904416647, entries=3807, filesize=64965359. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904428780 2011-09-13 02:00:28,943 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904212358 whose highest sequenceid is 772951 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315904212358 2011-09-13 02:00:36,195 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/.tmp/2644596471806230331 to hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/family/4245866439560704825 2011-09-13 02:00:36,257 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/family/4245866439560704825, entries=205370, sequenceid=799925, memsize=64.2m, filesize=28.6m 2011-09-13 02:00:37,058 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.2m for region usertable,user1772610920,1315898727287.77b5b2f51884554ebf951146d25f1bd2. in 3916ms, sequenceid=799925, compaction requested=true 2011-09-13 02:00:46,929 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/169b91a09b9b10a23602aaea5e8b9fc2/.tmp/3046893345723429739 to hdfs://c0309.xxxxx:17020/hbase/usertable/169b91a09b9b10a23602aaea5e8b9fc2/family/6069411880536292909 2011-09-13 02:00:46,968 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/169b91a09b9b10a23602aaea5e8b9fc2/family/6069411880536292909, entries=205870, sequenceid=801500, memsize=64.4m, filesize=28.7m 2011-09-13 02:00:46,972 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.4m for region usertable,user1901138600,1315898727288.169b91a09b9b10a23602aaea5e8b9fc2. in 1582ms, sequenceid=801500, compaction requested=true 2011-09-13 02:01:08,861 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 4 file(s) in family of usertable,user1649438560,1315898727286.29ec66424227dc3cbe3d8b2311764fc9.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/family/4466717696601045715, size=325.2m; total size for store is 890.5m 2011-09-13 02:01:08,861 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user1649438560,1315898727286.29ec66424227dc3cbe3d8b2311764fc9., storeName=family, fileCount=4, fileSize=325.2m, priority=2, time=3918654312820297; duration=1mins, 21sec 2011-09-13 02:01:08,862 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user1901138600,1315898727288.169b91a09b9b10a23602aaea5e8b9fc2. 2011-09-13 02:01:08,862 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 4 file(s) in family of usertable,user1901138600,1315898727288.169b91a09b9b10a23602aaea5e8b9fc2. into hdfs://c0309.xxxxx:17020/hbase/usertable/169b91a09b9b10a23602aaea5e8b9fc2/.tmp, seqid=801500, totalSize=322.0m 2011-09-13 02:01:48,845 WARN org.apache.hadoop.hbase.regionserver.wal.HLog: IPC Server handler 23 on 60020 took 1033 ms appending an edit to hlog; editcount=3847, len~=11.4k 2011-09-13 02:01:50,749 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/.tmp/7900377533412648341 to hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/family/1774412240971935362 2011-09-13 02:01:50,780 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/family/1774412240971935362, entries=206750, sequenceid=802689, memsize=64.7m, filesize=28.8m 2011-09-13 02:01:51,281 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 4 file(s) in family of usertable,user1901138600,1315898727288.169b91a09b9b10a23602aaea5e8b9fc2.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/169b91a09b9b10a23602aaea5e8b9fc2/family/6435153248387204207, size=322.0m; total size for store is 904.5m 2011-09-13 02:01:51,282 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user1901138600,1315898727288.169b91a09b9b10a23602aaea5e8b9fc2., storeName=family, fileCount=4, fileSize=322.0m, priority=2, time=3918748229359297; duration=42sec 2011-09-13 02:01:51,282 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user1397738520,1315898727283.d057127fe98e78ea72dc9be657e68a4a. 2011-09-13 02:01:51,282 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 4 file(s) in family of usertable,user1397738520,1315898727283.d057127fe98e78ea72dc9be657e68a4a. into hdfs://c0309.xxxxx:17020/hbase/usertable/d057127fe98e78ea72dc9be657e68a4a/.tmp, seqid=785289, totalSize=887.4m 2011-09-13 02:01:52,445 WARN org.apache.hadoop.hbase.regionserver.wal.HLog: IPC Server handler 9 on 60020 took 2042 ms appending an edit to hlog; editcount=3967, len~=15.7k 2011-09-13 02:01:52,660 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.7m for region usertable,user2029666280,1315898727289.8cb227d0fbbb2ad66e2f3d25cdfa0f89. in 8248ms, sequenceid=802689, compaction requested=true 2011-09-13 02:01:53,073 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904428780, entries=4256, filesize=72544994. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904512660 2011-09-13 02:01:53,074 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904232334 whose highest sequenceid is 776757 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315904232334 2011-09-13 02:01:57,584 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/.tmp/2576829131311074796 to hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/family/8298822990395663553 2011-09-13 02:01:57,646 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/family/8298822990395663553, entries=208530, sequenceid=804496, memsize=65.1m, filesize=28.8m 2011-09-13 02:01:57,647 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~65.1m for region usertable,user728323520,1315898727295.a623bd664ea942437bf6ae1073fb5560. in 2541ms, sequenceid=804496, compaction requested=true 2011-09-13 02:01:57,648 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user728323520,1315898727295.a623bd664ea942437bf6ae1073fb5560. 2011-09-13 02:01:57,648 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 3 file(s) in family of usertable,user728323520,1315898727295.a623bd664ea942437bf6ae1073fb5560. into hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/.tmp, seqid=804496, totalSize=90.1m 2011-09-13 02:02:05,113 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/766e5174f1140d766dd0af91cbd1d845/.tmp/4133287998358062804 to hdfs://c0309.xxxxx:17020/hbase/usertable/766e5174f1140d766dd0af91cbd1d845/family/5432725736450817615 2011-09-13 02:02:05,163 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/766e5174f1140d766dd0af91cbd1d845/family/5432725736450817615, entries=208200, sequenceid=805656, memsize=65.0m, filesize=28.8m 2011-09-13 02:02:06,127 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~65.0m for region usertable,user224923440,1315898727290.766e5174f1140d766dd0af91cbd1d845. in 8479ms, sequenceid=805656, compaction requested=true 2011-09-13 02:02:06,951 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904512660, entries=4504, filesize=76220474. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904526127 2011-09-13 02:02:08,140 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/.tmp/6079259732604126753 to hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/family/7020363705597465232 2011-09-13 02:02:08,186 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/family/7020363705597465232, entries=222670, sequenceid=808086, memsize=69.6m, filesize=31.0m 2011-09-13 02:02:09,449 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~69.6m for region usertable,user1022866120,1315898727279.f3cb8bb14102b43abbb2b45cd2fc4945. in 3322ms, sequenceid=808086, compaction requested=true 2011-09-13 02:02:12,293 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/.tmp/8104812779144701896 to hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/family/2499787219180927748 2011-09-13 02:02:12,333 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/family/2499787219180927748, entries=215520, sequenceid=810582, memsize=67.4m, filesize=30.0m 2011-09-13 02:02:12,442 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~67.4m for region usertable,user1146038480,1315898727281.f6d7a2cd3251b638e0f60e6e46e71a88. in 2993ms, sequenceid=810582, compaction requested=false 2011-09-13 02:02:12,583 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904526127, entries=4483, filesize=76086383. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904532442 2011-09-13 02:02:12,583 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904287828 whose highest sequenceid is 782012 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315904287828 2011-09-13 02:02:18,281 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 3 file(s) in family of usertable,user728323520,1315898727295.a623bd664ea942437bf6ae1073fb5560.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/family/6925552222843966063, size=90.1m; total size for store is 897.9m 2011-09-13 02:02:18,281 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user728323520,1315898727295.a623bd664ea942437bf6ae1073fb5560., storeName=family, fileCount=3, fileSize=90.1m, priority=2, time=3918818904969297; duration=20sec 2011-09-13 02:02:20,813 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/d057127fe98e78ea72dc9be657e68a4a/.tmp/6211229825344752593 to hdfs://c0309.xxxxx:17020/hbase/usertable/d057127fe98e78ea72dc9be657e68a4a/family/6100604291628628870 2011-09-13 02:02:20,867 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/d057127fe98e78ea72dc9be657e68a4a/family/6100604291628628870, entries=220180, sequenceid=812594, memsize=68.9m, filesize=30.7m 2011-09-13 02:02:20,906 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~68.9m for region usertable,user1397738520,1315898727283.d057127fe98e78ea72dc9be657e68a4a. in 8464ms, sequenceid=812594, compaction requested=false 2011-09-13 02:02:28,349 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/.tmp/8325868071849258628 to hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/family/4651744746077214506 2011-09-13 02:02:28,575 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/family/4651744746077214506, entries=214010, sequenceid=814582, memsize=66.9m, filesize=29.8m 2011-09-13 02:02:28,577 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~66.9m for region usertable,user1649438560,1315898727286.29ec66424227dc3cbe3d8b2311764fc9. in 7671ms, sequenceid=814582, compaction requested=false 2011-09-13 02:02:28,867 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904532442, entries=4159, filesize=70891679. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904548577 2011-09-13 02:02:28,868 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315904305213 whose highest sequenceid is 786395 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315904305213 -- 2011-09-13 03:22:09,954 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11327,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@66028287)","client":"172.29.81.104:60481","starttimems":1315909318626,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:22:10,004 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11482,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7c7ac9cb)","client":"172.29.81.115:36960","starttimems":1315909318521,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:22:17,481 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/.tmp/6143958488601306699 to hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/family/2410762954493688648 2011-09-13 03:22:17,513 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/family/2410762954493688648, entries=204800, sequenceid=1389926, memsize=64.0m, filesize=28.5m 2011-09-13 03:22:18,910 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.0m for region usertable,user1649438560,1315898727286.29ec66424227dc3cbe3d8b2311764fc9. in 3520ms, sequenceid=1389926, compaction requested=true 2011-09-13 03:22:19,337 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909314928, entries=3765, filesize=64376935. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909339159 2011-09-13 03:22:19,339 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909144928 whose highest sequenceid is 1364606 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315909144928 2011-09-13 03:22:19,352 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909172510 whose highest sequenceid is 1369761 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315909172510 2011-09-13 03:22:25,812 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909339159, entries=4014, filesize=68599818. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909344470 2011-09-13 03:22:26,681 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/8c36971138a4bfebe5aeb94c163f2a81/.tmp/4951163820172404908 to hdfs://c0309.xxxxx:17020/hbase/usertable/8c36971138a4bfebe5aeb94c163f2a81/family/6862581974323692936 2011-09-13 03:22:26,704 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/8c36971138a4bfebe5aeb94c163f2a81/family/6862581974323692936, entries=206260, sequenceid=1394859, memsize=64.5m, filesize=28.7m 2011-09-13 03:22:26,796 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.5m for region usertable,user1274566160,1315898727282.8c36971138a4bfebe5aeb94c163f2a81. in 2254ms, sequenceid=1394859, compaction requested=true 2011-09-13 03:22:30,739 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/.tmp/6849301759380026107 to hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/family/4847541390409534955 2011-09-13 03:22:30,893 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/family/4847541390409534955, entries=206160, sequenceid=1395560, memsize=64.3m, filesize=28.5m 2011-09-13 03:22:30,900 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.3m for region usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d. in 4104ms, sequenceid=1395560, compaction requested=true 2011-09-13 03:22:33,475 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/.tmp/6227021282012378096 to hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/family/1202425479753407405 2011-09-13 03:22:33,744 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/family/1202425479753407405, entries=221100, sequenceid=1397261, memsize=69.0m, filesize=30.6m 2011-09-13 03:22:33,749 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~69.0m for region usertable,user728323520,1315898727295.a623bd664ea942437bf6ae1073fb5560. in 2848ms, sequenceid=1397261, compaction requested=true 2011-09-13 03:22:33,750 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user728323520,1315898727295.a623bd664ea942437bf6ae1073fb5560. 2011-09-13 03:22:33,750 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 4 file(s) in family of usertable,user728323520,1315898727295.a623bd664ea942437bf6ae1073fb5560. into hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/.tmp, seqid=1397261, totalSize=346.9m 2011-09-13 03:22:39,851 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/.tmp/3744599207002088432 to hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/family/8359715519758850952 2011-09-13 03:22:39,890 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/family/8359715519758850952, entries=221440, sequenceid=1397509, memsize=69.1m, filesize=30.6m 2011-09-13 03:22:40,498 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~69.1m for region usertable,user980023560,1315898727298.a4521b468fb6ac94dd495ee2db515ae0. in 6748ms, sequenceid=1397509, compaction requested=true 2011-09-13 03:22:40,499 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user980023560,1315898727298.a4521b468fb6ac94dd495ee2db515ae0. 2011-09-13 03:22:40,499 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 3 file(s) in family of usertable,user980023560,1315898727298.a4521b468fb6ac94dd495ee2db515ae0. into hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/.tmp, seqid=1397509, totalSize=90.1m 2011-09-13 03:22:40,736 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909344470, entries=3819, filesize=64458845. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909360498 2011-09-13 03:22:47,340 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/.tmp/1572756159433513036 to hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/family/5723561466813807612 2011-09-13 03:22:47,397 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/family/5723561466813807612, entries=220440, sequenceid=1398735, memsize=68.8m, filesize=30.5m 2011-09-13 03:22:47,399 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~68.8m for region usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd. in 6900ms, sequenceid=1398735, compaction requested=true 2011-09-13 03:22:49,302 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/.tmp/5157756983548695534 to hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/family/6469244982165292612 2011-09-13 03:22:49,389 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/family/6469244982165292612, entries=228500, sequenceid=1401069, memsize=71.5m, filesize=31.8m 2011-09-13 03:22:49,391 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~71.5m for region usertable,user1022866120,1315898727279.f3cb8bb14102b43abbb2b45cd2fc4945. in 1992ms, sequenceid=1401069, compaction requested=true 2011-09-13 03:22:53,639 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/.tmp/6234314149785698509 to hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/family/6900207983304840996 2011-09-13 03:22:53,696 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/family/6900207983304840996, entries=221070, sequenceid=1401599, memsize=69.1m, filesize=30.8m 2011-09-13 03:22:53,698 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~69.1m for region usertable,user1146038480,1315898727281.f6d7a2cd3251b638e0f60e6e46e71a88. in 4306ms, sequenceid=1401599, compaction requested=true 2011-09-13 03:22:58,008 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/d057127fe98e78ea72dc9be657e68a4a/.tmp/8491627372668174645 to hdfs://c0309.xxxxx:17020/hbase/usertable/d057127fe98e78ea72dc9be657e68a4a/family/1390610362222217593 2011-09-13 03:22:58,062 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/d057127fe98e78ea72dc9be657e68a4a/family/1390610362222217593, entries=208630, sequenceid=1401847, memsize=65.2m, filesize=29.1m 2011-09-13 03:22:58,072 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~65.2m for region usertable,user1397738520,1315898727283.d057127fe98e78ea72dc9be657e68a4a. in 4373ms, sequenceid=1401847, compaction requested=true 2011-09-13 03:22:58,137 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909360498, entries=4273, filesize=72939649. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909378072 2011-09-13 03:22:58,139 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909192040 whose highest sequenceid is 1373767 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315909192040 2011-09-13 03:22:58,161 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909239160 whose highest sequenceid is 1378056 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315909239160 2011-09-13 03:23:04,086 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 3 file(s) in family of usertable,user980023560,1315898727298.a4521b468fb6ac94dd495ee2db515ae0.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/family/1661386191397716109, size=90.1m; total size for store is 1.5g 2011-09-13 03:23:04,087 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user980023560,1315898727298.a4521b468fb6ac94dd495ee2db515ae0., storeName=family, fileCount=3, fileSize=90.1m, priority=2, time=3923661756153297; duration=23sec 2011-09-13 03:23:04,139 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/.tmp/7045913442422156167 to hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/family/7768571669536598121 2011-09-13 03:23:04,185 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/family/7768571669536598121, entries=206920, sequenceid=1403372, memsize=64.7m, filesize=28.8m 2011-09-13 03:23:04,193 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.7m for region usertable,user1772610920,1315898727287.77b5b2f51884554ebf951146d25f1bd2. in 2261ms, sequenceid=1403372, compaction requested=true 2011-09-13 03:23:29,829 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/169b91a09b9b10a23602aaea5e8b9fc2/.tmp/71313572870804822 to hdfs://c0309.xxxxx:17020/hbase/usertable/169b91a09b9b10a23602aaea5e8b9fc2/family/7582072035501018142 2011-09-13 03:23:29,884 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/169b91a09b9b10a23602aaea5e8b9fc2/family/7582072035501018142, entries=204890, sequenceid=1404647, memsize=64.1m, filesize=28.5m 2011-09-13 03:23:30,003 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.1m for region usertable,user1901138600,1315898727288.169b91a09b9b10a23602aaea5e8b9fc2. in 7190ms, sequenceid=1404647, compaction requested=true 2011-09-13 03:23:44,931 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 4 file(s) in family of usertable,user728323520,1315898727295.a623bd664ea942437bf6ae1073fb5560.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/family/6336430120895989246, size=346.9m; total size for store is 1.5g 2011-09-13 03:23:44,931 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user728323520,1315898727295.a623bd664ea942437bf6ae1073fb5560., storeName=family, fileCount=4, fileSize=346.9m, priority=2, time=3923655006967297; duration=1mins, 11sec 2011-09-13 03:23:44,931 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user1022866120,1315898727279.f3cb8bb14102b43abbb2b45cd2fc4945. 2011-09-13 03:23:44,931 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 4 file(s) in family of usertable,user1022866120,1315898727279.f3cb8bb14102b43abbb2b45cd2fc4945. into hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/.tmp, seqid=1401069, totalSize=182.0m 2011-09-13 03:24:10,756 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 4 file(s) in family of usertable,user1022866120,1315898727279.f3cb8bb14102b43abbb2b45cd2fc4945.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/family/3654927095782348627, size=182.0m; total size for store is 1.5g 2011-09-13 03:24:10,756 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user1022866120,1315898727279.f3cb8bb14102b43abbb2b45cd2fc4945., storeName=family, fileCount=4, fileSize=182.0m, priority=1, time=3923670649053297; duration=25sec 2011-09-13 03:24:10,757 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user1146038480,1315898727281.f6d7a2cd3251b638e0f60e6e46e71a88. 2011-09-13 03:24:10,757 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 4 file(s) in family of usertable,user1146038480,1315898727281.f6d7a2cd3251b638e0f60e6e46e71a88. into hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/.tmp, seqid=1401599, totalSize=182.0m 2011-09-13 03:24:39,953 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 4 file(s) in family of usertable,user1146038480,1315898727281.f6d7a2cd3251b638e0f60e6e46e71a88.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/family/8563877190719595637, size=182.0m; total size for store is 1.5g 2011-09-13 03:24:39,953 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user1146038480,1315898727281.f6d7a2cd3251b638e0f60e6e46e71a88., storeName=family, fileCount=4, fileSize=182.0m, priority=1, time=3923674956081297; duration=29sec 2011-09-13 03:24:39,954 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user1397738520,1315898727283.d057127fe98e78ea72dc9be657e68a4a. 2011-09-13 03:24:39,954 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 5 file(s) in family of usertable,user1397738520,1315898727283.d057127fe98e78ea72dc9be657e68a4a. into hdfs://c0309.xxxxx:17020/hbase/usertable/d057127fe98e78ea72dc9be657e68a4a/.tmp, seqid=1401847, totalSize=693.9m 2011-09-13 03:25:05,097 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909378072, entries=3753, filesize=64284443. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909504812 2011-09-13 03:25:05,097 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909256318 whose highest sequenceid is 1381801 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315909256318 2011-09-13 03:25:13,990 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/.tmp/1306352367264402214 to hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/family/8385554065938044307 2011-09-13 03:25:14,038 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/family/8385554065938044307, entries=204980, sequenceid=1407198, memsize=64.1m, filesize=28.6m 2011-09-13 03:25:14,776 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.1m for region usertable,user2029666280,1315898727289.8cb227d0fbbb2ad66e2f3d25cdfa0f89. in 7956ms, sequenceid=1407198, compaction requested=true 2011-09-13 03:25:21,718 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/766e5174f1140d766dd0af91cbd1d845/.tmp/8870854879729905885 to hdfs://c0309.xxxxx:17020/hbase/usertable/766e5174f1140d766dd0af91cbd1d845/family/5784426002732966271 2011-09-13 03:25:21,754 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/766e5174f1140d766dd0af91cbd1d845/family/5784426002732966271, entries=207510, sequenceid=1410006, memsize=64.8m, filesize=28.7m 2011-09-13 03:25:21,985 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.8m for region usertable,user224923440,1315898727290.766e5174f1140d766dd0af91cbd1d845. in 4581ms, sequenceid=1410006, compaction requested=true 2011-09-13 03:25:22,207 WARN org.apache.hadoop.hdfs.DFSClient: Error while syncing java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1041) 2011-09-13 03:25:22,207 FATAL org.apache.hadoop.hbase.regionserver.wal.HLog: Could not sync. Requesting close of hlog java.io.IOException: Reflection at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:194) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1041) Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) ... 3 more Caused by: java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) ... 7 more 2011-09-13 03:25:22,208 ERROR org.apache.hadoop.hbase.regionserver.wal.HLog: Error while syncing, requesting close of hlog java.io.IOException: Reflection at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:194) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1041) Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) ... 3 more Caused by: java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) ... 7 more 2011-09-13 03:25:22,208 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909504812, entries=4057, filesize=69286717. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909521984 2011-09-13 03:25:22,210 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909305702 whose highest sequenceid is 1387078 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315909305702 2011-09-13 03:25:25,242 WARN org.apache.hadoop.hbase.regionserver.wal.HLog: IPC Server handler 85 on 60020 took 3012 ms appending an edit to hlog; editcount=82, len~=12.8k 2011-09-13 03:25:26,074 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10403,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2b46309d)","client":"172.29.81.93:56239","starttimems":1315909515669,"queuetimems":5199,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:26,082 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10162,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3f800376)","client":"172.29.81.98:40144","starttimems":1315909515919,"queuetimems":3373,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:26,084 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10162,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3f2b5863)","client":"172.29.81.115:42912","starttimems":1315909515921,"queuetimems":3313,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:26,289 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10570,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@21065524)","client":"172.29.81.99:36509","starttimems":1315909515718,"queuetimems":4075,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:26,337 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10461,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4070a20d)","client":"172.29.81.117:57673","starttimems":1315909515874,"queuetimems":3351,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:26,347 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10078,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@6757e6e6)","client":"172.29.81.119:40856","starttimems":1315909516268,"queuetimems":3373,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:29,210 WARN org.apache.hadoop.hbase.regionserver.wal.HLog: IPC Server handler 33 on 60020 took 2636 ms appending an edit to hlog; editcount=437, len~=11.3k 2011-09-13 03:25:29,520 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12001,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@6ec2dba)","client":"172.29.81.113:37759","starttimems":1315909517518,"queuetimems":4601,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,227 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11838,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2dc99f56)","client":"172.29.81.108:35203","starttimems":1315909518388,"queuetimems":5350,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,252 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11874,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@41c1f23c)","client":"172.29.81.120:57402","starttimems":1315909518377,"queuetimems":5402,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,286 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11893,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@14426c66)","client":"172.29.81.98:40144","starttimems":1315909518392,"queuetimems":5293,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,443 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12879,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@77348cae)","client":"172.29.81.114:36887","starttimems":1315909517563,"queuetimems":4636,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,670 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12294,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@199ebefd)","client":"172.29.81.103:52453","starttimems":1315909518374,"queuetimems":5429,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,687 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13107,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@392ba08b)","client":"172.29.81.109:60717","starttimems":1315909517579,"queuetimems":4647,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,693 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12304,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@33789fbf)","client":"172.29.81.113:37759","starttimems":1315909518388,"queuetimems":5348,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,705 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13069,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@141d005b)","client":"172.29.81.96:39001","starttimems":1315909517635,"queuetimems":4701,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,713 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12312,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@c013c64)","client":"172.29.81.95:55253","starttimems":1315909518400,"queuetimems":5260,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,713 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12334,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5f58063b)","client":"172.29.81.116:37302","starttimems":1315909518378,"queuetimems":5385,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,714 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12325,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5204c148)","client":"172.29.81.106:36055","starttimems":1315909518387,"queuetimems":5381,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,718 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12823,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@939cf56)","client":"172.29.81.100:36213","starttimems":1315909517894,"queuetimems":4960,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,725 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12333,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@35696c98)","client":"172.29.81.110:58839","starttimems":1315909518391,"queuetimems":5345,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,729 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12345,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@21b47d8c)","client":"172.29.81.118:56133","starttimems":1315909518383,"queuetimems":5384,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,734 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12331,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@10aa751f)","client":"172.29.81.93:56239","starttimems":1315909518402,"queuetimems":5107,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,737 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12842,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4149a4b5)","client":"172.29.81.107:52421","starttimems":1315909517894,"queuetimems":4958,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,751 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12348,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4f405838)","client":"172.29.81.107:52421","starttimems":1315909518402,"queuetimems":5074,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,751 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12356,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5c787358)","client":"172.29.81.101:36264","starttimems":1315909518394,"queuetimems":5279,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,764 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12383,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2d178d9c)","client":"172.29.81.102:53861","starttimems":1315909518381,"queuetimems":5384,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,767 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13232,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@44817f95)","client":"172.29.81.99:36509","starttimems":1315909517534,"queuetimems":4613,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,768 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12393,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@11df3256)","client":"172.29.81.96:39001","starttimems":1315909518374,"queuetimems":5406,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,800 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12390,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@69e8ca38)","client":"172.29.81.119:40856","starttimems":1315909518409,"queuetimems":5081,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,811 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12408,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5e7128a3)","client":"172.29.81.97:55701","starttimems":1315909518402,"queuetimems":5164,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,829 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12427,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@35a276c0)","client":"172.29.81.94:46169","starttimems":1315909518401,"queuetimems":5166,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,881 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12493,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@79fe918d)","client":"172.29.81.114:36887","starttimems":1315909518387,"queuetimems":5368,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,926 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12525,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2eaa6bf)","client":"172.29.81.95:55253","starttimems":1315909518400,"queuetimems":5230,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:30,928 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12539,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3cba764d)","client":"172.29.81.117:57673","starttimems":1315909518388,"queuetimems":5349,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:31,209 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12239,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7057f64a)","client":"172.29.81.110:58839","starttimems":1315909518969,"queuetimems":5455,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:31,493 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11250,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@60d7950c)","client":"172.29.81.96:39001","starttimems":1315909520242,"queuetimems":6587,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:31,523 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12598,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@20d5019c)","client":"172.29.81.118:56133","starttimems":1315909518924,"queuetimems":5425,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:31,524 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12605,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@479d6f4c)","client":"172.29.81.109:60717","starttimems":1315909518918,"queuetimems":5520,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:31,525 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13105,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@30f60b4f)","client":"172.29.81.120:57402","starttimems":1315909518419,"queuetimems":5071,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:31,525 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13131,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@696088a5)","client":"172.29.81.101:36264","starttimems":1315909518393,"queuetimems":5286,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:31,543 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12621,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@378e1e2a)","client":"172.29.81.116:37302","starttimems":1315909518921,"queuetimems":5479,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:31,543 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12621,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@63b7cb73)","client":"172.29.81.99:36509","starttimems":1315909518921,"queuetimems":5522,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:31,549 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12575,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@dbc186e)","client":"172.29.81.103:52453","starttimems":1315909518973,"queuetimems":5454,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:31,570 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13175,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2e83f5e)","client":"172.29.81.100:36213","starttimems":1315909518394,"queuetimems":5286,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:31,573 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13151,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@fd53a2c)","client":"172.29.81.106:36055","starttimems":1315909518421,"queuetimems":5069,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,021 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13609,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4dc82bd9)","client":"172.29.81.112:54573","starttimems":1315909518411,"queuetimems":5072,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,022 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13632,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@229e6828)","client":"172.29.81.104:60729","starttimems":1315909518388,"queuetimems":5343,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,022 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13607,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@10b6db74)","client":"172.29.81.94:46169","starttimems":1315909518414,"queuetimems":5073,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,026 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13611,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@48f7f0a1)","client":"172.29.81.111:41541","starttimems":1315909518414,"queuetimems":5073,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,027 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12003,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1e1d1fc5)","client":"172.29.81.111:41541","starttimems":1315909520023,"queuetimems":6411,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,030 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11787,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@554194fc)","client":"172.29.81.114:36887","starttimems":1315909520242,"queuetimems":6560,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,059 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13143,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@367b9d98)","client":"172.29.81.102:53861","starttimems":1315909518916,"queuetimems":5539,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,066 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12824,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@303f7144)","client":"172.29.81.116:37302","starttimems":1315909519241,"queuetimems":5643,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,068 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13109,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@76ecc2c6)","client":"172.29.81.105:42440","starttimems":1315909518957,"queuetimems":5449,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,073 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12831,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5982efa9)","client":"172.29.81.108:35203","starttimems":1315909519241,"queuetimems":5658,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,083 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11838,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@777dc8e0)","client":"172.29.81.107:52421","starttimems":1315909520244,"queuetimems":6558,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,096 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11127,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7d3b76fe)","client":"172.29.81.96:39001","starttimems":1315909520968,"queuetimems":7185,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,096 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13178,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4b015ded)","client":"172.29.81.97:55701","starttimems":1315909518917,"queuetimems":5528,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,329 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12089,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@72690820)","client":"172.29.81.95:55253","starttimems":1315909520239,"queuetimems":6608,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,347 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11273,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@39227f29)","client":"172.29.81.113:37759","starttimems":1315909521073,"queuetimems":7060,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,370 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11832,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@50ea1e22)","client":"172.29.81.112:54573","starttimems":1315909520537,"queuetimems":6834,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,373 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11262,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@66f911fa)","client":"172.29.81.103:52453","starttimems":1315909521110,"queuetimems":7045,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,380 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11269,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@664ceeec)","client":"172.29.81.104:60729","starttimems":1315909521110,"queuetimems":7096,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,380 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11397,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@434b39c4)","client":"172.29.81.100:36213","starttimems":1315909520982,"queuetimems":7124,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,393 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11525,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@78a4b2ee)","client":"172.29.81.98:40144","starttimems":1315909520867,"queuetimems":7085,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,393 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11565,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4df6523)","client":"172.29.81.94:46169","starttimems":1315909520827,"queuetimems":7121,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,394 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11566,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@6f1e3852)","client":"172.29.81.115:42912","starttimems":1315909520827,"queuetimems":7098,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,406 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11356,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@14b5088)","client":"172.29.81.102:53861","starttimems":1315909521049,"queuetimems":7098,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,410 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12170,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@988a088)","client":"172.29.81.101:36264","starttimems":1315909520239,"queuetimems":6607,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,418 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11270,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1fc80d0d)","client":"172.29.81.111:41541","starttimems":1315909521147,"queuetimems":6849,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,426 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11304,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3126af0d)","client":"172.29.81.110:58839","starttimems":1315909521121,"queuetimems":6858,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,439 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11389,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@294ddb89)","client":"172.29.81.93:56239","starttimems":1315909521049,"queuetimems":7097,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,444 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11333,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@41df23f9)","client":"172.29.81.119:40856","starttimems":1315909521110,"queuetimems":7000,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,444 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11382,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3c076b4d)","client":"172.29.81.93:56239","starttimems":1315909521061,"queuetimems":7056,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,445 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12202,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3b8edaf2)","client":"172.29.81.117:57673","starttimems":1315909520241,"queuetimems":6604,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,456 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11308,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@18c0dda2)","client":"172.29.81.105:42440","starttimems":1315909521147,"queuetimems":6877,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,461 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11350,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@71916d53)","client":"172.29.81.115:42912","starttimems":1315909521110,"queuetimems":7007,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,463 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11924,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2a09e65)","client":"172.29.81.104:60729","starttimems":1315909520537,"queuetimems":6843,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:25:32,493 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11298,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@e3c099e)","client":"172.29.81.118:56133","starttimems":1315909521194,"queuetimems":6870,"class":"HRegionServer","responsesize":0,"method":"multi"} -- 2011-09-13 03:29:26,026 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":14125,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@242e3717)","client":"172.29.81.96:59385","starttimems":1315909751900,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:26,041 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":14130,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1d0071d2)","client":"172.29.81.95:44021","starttimems":1315909751910,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:26,043 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":14112,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@14e1862c)","client":"172.29.81.103:60665","starttimems":1315909751929,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:26,043 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":14114,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@622fbd69)","client":"172.29.81.111:43279","starttimems":1315909751928,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:26,043 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":14165,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2c3c6adb)","client":"172.29.81.112:46050","starttimems":1315909751877,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:26,049 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":14136,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5e8d1786)","client":"172.29.81.114:35844","starttimems":1315909751912,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:26,049 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.9m for region usertable,user1526266200,1315898727285.50d20bc0e631f85cd8b961fcf983914a. in 14498ms, sequenceid=1437772, compaction requested=true 2011-09-13 03:29:26,059 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":14132,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@39843b89)","client":"172.29.81.95:44021","starttimems":1315909751926,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:26,068 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13925,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@663aaee9)","client":"172.29.81.98:41446","starttimems":1315909752142,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:26,084 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13962,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7507618f)","client":"172.29.81.98:41446","starttimems":1315909752121,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:26,088 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":14066,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2190953a)","client":"172.29.81.107:34023","starttimems":1315909752021,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:26,090 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":14010,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7e1e30c5)","client":"172.29.81.114:35844","starttimems":1315909752080,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:26,095 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":14014,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@959378e)","client":"172.29.81.117:58003","starttimems":1315909752080,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:26,099 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":14125,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@652dbf2d)","client":"172.29.81.92:60640","starttimems":1315909751973,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:26,103 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":14071,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7dbf1a0e)","client":"172.29.81.101:36663","starttimems":1315909752031,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:26,112 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13841,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3958d086)","client":"172.29.81.114:35844","starttimems":1315909752270,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:27,175 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":14755,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@24073573)","client":"172.29.81.98:41446","starttimems":1315909752418,"queuetimems":92,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:27,249 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13235,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@8d06bfd)","client":"172.29.81.112:46050","starttimems":1315909754013,"queuetimems":1627,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:27,250 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":14804,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2b61b390)","client":"172.29.81.120:35860","starttimems":1315909752445,"queuetimems":88,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:28,029 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13930,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@6c9bb63f)","client":"172.29.81.96:59385","starttimems":1315909754098,"queuetimems":1708,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:28,116 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":14006,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@271a35ab)","client":"172.29.81.114:35844","starttimems":1315909754109,"queuetimems":1648,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:28,142 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13889,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5597b3a8)","client":"172.29.81.101:36663","starttimems":1315909754251,"queuetimems":1654,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:28,990 WARN org.apache.hadoop.hdfs.DFSClient: Error while syncing java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1041) 2011-09-13 03:29:28,990 WARN org.apache.hadoop.hdfs.DFSClient: Error while syncing java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:1016) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1757) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1627) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2910) at sun.reflect.GeneratedMethodAccessor25.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:348) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1226) 2011-09-13 03:29:28,990 FATAL org.apache.hadoop.hbase.regionserver.wal.HLog: Could not sync. Requesting close of hlog java.io.IOException: Reflection at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:194) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1041) Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) ... 3 more Caused by: java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) ... 7 more 2011-09-13 03:29:28,990 FATAL org.apache.hadoop.hbase.regionserver.wal.HLog: Could not sync. Requesting close of hlog java.io.IOException: Reflection at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:194) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:1016) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1757) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1627) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2910) at sun.reflect.GeneratedMethodAccessor25.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:348) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1226) Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) ... 11 more Caused by: java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) ... 15 more 2011-09-13 03:29:28,991 ERROR org.apache.hadoop.hbase.regionserver.wal.HLog: Error while syncing, requesting close of hlog java.io.IOException: Reflection at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:194) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1041) Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) ... 3 more Caused by: java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) ... 7 more 2011-09-13 03:29:29,016 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909728806, entries=3756, filesize=64385036. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909768115 2011-09-13 03:29:29,016 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909504812 whose highest sequenceid is 1410759 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315909504812 2011-09-13 03:29:29,024 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":14776,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@48509d36)","client":"172.29.81.101:36663","starttimems":1315909754246,"queuetimems":1716,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,116 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909521984 whose highest sequenceid is 1414520 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315909521984 2011-09-13 03:29:29,156 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":14783,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3ac2c5cd)","client":"172.29.81.107:34023","starttimems":1315909754371,"queuetimems":1758,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,172 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13501,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4fcd0fb5)","client":"172.29.81.106:59507","starttimems":1315909755670,"queuetimems":3052,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,202 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10332,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2efaf51c)","client":"172.29.81.101:36663","starttimems":1315909758869,"queuetimems":6184,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,204 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11869,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@49ec36cc)","client":"172.29.81.101:36663","starttimems":1315909757334,"queuetimems":4702,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,279 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10452,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@44543f9)","client":"172.29.81.100:55923","starttimems":1315909758826,"queuetimems":6184,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,282 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10071,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@6d8f4000)","client":"172.29.81.101:36663","starttimems":1315909759210,"queuetimems":6489,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,306 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10440,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@517c6f5a)","client":"172.29.81.101:36663","starttimems":1315909758865,"queuetimems":6201,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,528 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10181,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2b299f91)","client":"172.29.81.114:35844","starttimems":1315909759346,"queuetimems":6412,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,528 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10237,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@39aafeba)","client":"172.29.81.101:36663","starttimems":1315909759291,"queuetimems":6497,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,562 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10680,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@51f38f37)","client":"172.29.81.101:36663","starttimems":1315909758881,"queuetimems":6177,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,594 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10736,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2d864fef)","client":"172.29.81.114:35844","starttimems":1315909758857,"queuetimems":6209,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,594 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10333,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4ce9eb42)","client":"172.29.81.101:36663","starttimems":1315909759260,"queuetimems":6504,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,632 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10386,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4fd4b570)","client":"172.29.81.101:36663","starttimems":1315909759245,"queuetimems":6508,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,649 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10379,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@70ea667)","client":"172.29.81.114:35844","starttimems":1315909759269,"queuetimems":6509,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,652 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10364,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5843bfaf)","client":"172.29.81.101:36663","starttimems":1315909759287,"queuetimems":6515,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,652 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10357,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@47dda76e)","client":"172.29.81.114:35844","starttimems":1315909759294,"queuetimems":6464,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,662 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10382,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3bed3c27)","client":"172.29.81.96:59385","starttimems":1315909759279,"queuetimems":6518,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,669 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10377,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@411d1bde)","client":"172.29.81.108:40446","starttimems":1315909759291,"queuetimems":6491,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,670 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10379,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1d4b0ba0)","client":"172.29.81.101:36663","starttimems":1315909759291,"queuetimems":6485,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,673 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10313,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@586ad86b)","client":"172.29.81.114:35844","starttimems":1315909759359,"queuetimems":6362,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,677 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10300,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4e8c7df8)","client":"172.29.81.114:35844","starttimems":1315909759376,"queuetimems":6302,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,682 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10309,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7df3d84c)","client":"172.29.81.114:35844","starttimems":1315909759372,"queuetimems":6334,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,685 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10317,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@77a5b142)","client":"172.29.81.114:35844","starttimems":1315909759367,"queuetimems":6350,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,685 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10396,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1f18a16c)","client":"172.29.81.101:36663","starttimems":1315909759288,"queuetimems":6506,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,693 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10361,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7ad5261a)","client":"172.29.81.114:35844","starttimems":1315909759331,"queuetimems":6440,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,693 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10353,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3fb8b3f6)","client":"172.29.81.111:43279","starttimems":1315909759339,"queuetimems":6438,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,702 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10346,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2618deaa)","client":"172.29.81.114:35844","starttimems":1315909759355,"queuetimems":6390,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,702 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10335,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2ac27e70)","client":"172.29.81.114:35844","starttimems":1315909759366,"queuetimems":6355,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,709 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10340,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@27799149)","client":"172.29.81.114:35844","starttimems":1315909759368,"queuetimems":6340,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,710 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10334,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1e3163de)","client":"172.29.81.114:35844","starttimems":1315909759375,"queuetimems":6317,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,712 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10329,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1792d77)","client":"172.29.81.114:35844","starttimems":1315909759382,"queuetimems":6269,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,935 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10552,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@48c5145e)","client":"172.29.81.114:35844","starttimems":1315909759382,"queuetimems":6265,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,936 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10563,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2bc4ca5)","client":"172.29.81.114:35844","starttimems":1315909759372,"queuetimems":6325,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,936 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10560,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@184be60a)","client":"172.29.81.96:59385","starttimems":1315909759375,"queuetimems":6307,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:29,964 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10461,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@6c59a13b)","client":"172.29.81.118:56335","starttimems":1315909759502,"queuetimems":6370,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:30,239 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10856,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@34ea106a)","client":"172.29.81.114:35844","starttimems":1315909759382,"queuetimems":6261,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:30,246 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10753,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@298df516)","client":"172.29.81.114:35844","starttimems":1315909759492,"queuetimems":6366,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:30,246 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10719,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@26d58caa)","client":"172.29.81.114:35844","starttimems":1315909759526,"queuetimems":6394,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:30,295 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10916,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2f20aad8)","client":"172.29.81.114:35844","starttimems":1315909759378,"queuetimems":6284,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:30,672 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10092,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@455fb74f)","client":"172.29.81.114:35844","starttimems":1315909760579,"queuetimems":7420,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:29:37,346 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/.tmp/7912557644654635580 to hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/family/4617887256625612983 2011-09-13 03:29:37,408 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/family/4617887256625612983, entries=206250, sequenceid=1440718, memsize=64.5m, filesize=28.7m 2011-09-13 03:29:38,979 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.5m for region usertable,user1649438560,1315898727286.29ec66424227dc3cbe3d8b2311764fc9. in 6981ms, sequenceid=1440718, compaction requested=true 2011-09-13 03:29:38,979 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user1649438560,1315898727286.29ec66424227dc3cbe3d8b2311764fc9. 2011-09-13 03:29:38,980 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 3 file(s) in family of usertable,user1649438560,1315898727286.29ec66424227dc3cbe3d8b2311764fc9. into hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/.tmp, seqid=1440718, totalSize=86.1m 2011-09-13 03:29:39,669 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909768115, entries=3775, filesize=64152847. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909779170 2011-09-13 03:29:39,670 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909549318 whose highest sequenceid is 1418309 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315909549318 2011-09-13 03:29:50,294 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/8c36971138a4bfebe5aeb94c163f2a81/.tmp/5535615880465826838 to hdfs://c0309.xxxxx:17020/hbase/usertable/8c36971138a4bfebe5aeb94c163f2a81/family/7548983711867958474 2011-09-13 03:29:50,478 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/8c36971138a4bfebe5aeb94c163f2a81/family/7548983711867958474, entries=209240, sequenceid=1445880, memsize=65.4m, filesize=29.1m 2011-09-13 03:29:50,483 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~65.4m for region usertable,user1274566160,1315898727282.8c36971138a4bfebe5aeb94c163f2a81. in 3036ms, sequenceid=1445880, compaction requested=true 2011-09-13 03:29:50,882 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 3 file(s) in family of usertable,user1649438560,1315898727286.29ec66424227dc3cbe3d8b2311764fc9.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/family/1726482498742547020, size=86.1m; total size for store is 1.6g 2011-09-13 03:29:50,882 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user1649438560,1315898727286.29ec66424227dc3cbe3d8b2311764fc9., storeName=family, fileCount=3, fileSize=86.1m, priority=1, time=3924080236789297; duration=11sec 2011-09-13 03:29:55,711 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/.tmp/7655928679511731149 to hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/family/5380653054475964833 2011-09-13 03:29:55,735 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/family/5380653054475964833, entries=207020, sequenceid=1446764, memsize=64.6m, filesize=28.6m 2011-09-13 03:29:59,003 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.6m for region usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d. in 8520ms, sequenceid=1446764, compaction requested=true 2011-09-13 03:29:59,084 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909779170, entries=4064, filesize=69139884. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909799003 2011-09-13 03:29:59,085 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909570574 whose highest sequenceid is 1423313 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315909570574 2011-09-13 03:30:12,569 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/.tmp/8312478943733150050 to hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/family/2879399320520306524 2011-09-13 03:30:12,600 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/family/2879399320520306524, entries=207680, sequenceid=1448726, memsize=64.8m, filesize=28.7m 2011-09-13 03:30:12,708 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.8m for region usertable,user728323520,1315898727295.a623bd664ea942437bf6ae1073fb5560. in 7470ms, sequenceid=1448726, compaction requested=true 2011-09-13 03:30:18,367 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909799003, entries=3746, filesize=64238458. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909818264 2011-09-13 03:30:18,368 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909598193 whose highest sequenceid is 1428170 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315909598193 2011-09-13 03:31:17,623 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/.tmp/9125410028640654528 to hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/family/7666486072975543237 2011-09-13 03:31:17,696 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/family/7666486072975543237, entries=205310, sequenceid=1453074, memsize=64.2m, filesize=28.6m 2011-09-13 03:31:17,698 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.2m for region usertable,user1022866120,1315898727279.f3cb8bb14102b43abbb2b45cd2fc4945. in 2369ms, sequenceid=1453074, compaction requested=true 2011-09-13 03:31:19,634 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/.tmp/4259829405856076062 to hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/family/8363512844568917684 2011-09-13 03:31:19,703 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/family/8363512844568917684, entries=210380, sequenceid=1453768, memsize=65.6m, filesize=29.1m 2011-09-13 03:31:19,705 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~65.6m for region usertable,user980023560,1315898727298.a4521b468fb6ac94dd495ee2db515ae0. in 2007ms, sequenceid=1453768, compaction requested=true 2011-09-13 03:31:28,055 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/.tmp/4443357428523523956 to hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/family/2175088030507932993 2011-09-13 03:31:28,514 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/family/2175088030507932993, entries=205350, sequenceid=1453811, memsize=64.1m, filesize=28.4m 2011-09-13 03:31:28,519 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.1m for region usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd. in 8814ms, sequenceid=1453811, compaction requested=true 2011-09-13 03:31:28,520 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd. 2011-09-13 03:31:28,520 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 4 file(s) in family of usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd. into hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/.tmp, seqid=1453811, totalSize=678.8m 2011-09-13 03:31:28,946 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909818264, entries=4030, filesize=68906908. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315909888519 2011-09-13 03:31:30,623 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/.tmp/1067439917016736034 to hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/family/1276571755648702625 2011-09-13 03:31:30,679 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/family/1276571755648702625, entries=213890, sequenceid=1455271, memsize=66.9m, filesize=29.8m 2011-09-13 03:31:30,968 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~66.9m for region usertable,user1146038480,1315898727281.f6d7a2cd3251b638e0f60e6e46e71a88. in 2448ms, sequenceid=1455271, compaction requested=true 2011-09-13 03:31:46,105 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10116,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2a7db197)","client":"172.29.81.94:43687","starttimems":1315909895988,"queuetimems":307,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 03:31:46,609 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/.tmp/7085753684399513361 to hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/family/2492911328122123394 -- 2011-09-13 04:05:04,386 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":15807,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@74d433aa)","client":"172.29.81.92:59103","starttimems":1315911888578,"queuetimems":13908,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:04,683 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16094,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@dc270bd)","client":"172.29.81.104:50755","starttimems":1315911888588,"queuetimems":13649,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:04,683 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16098,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7f73d572)","client":"172.29.81.97:46852","starttimems":1315911888584,"queuetimems":13677,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:04,905 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16332,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@46d33d82)","client":"172.29.81.107:50949","starttimems":1315911888572,"queuetimems":13978,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:04,931 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16350,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@141f41c3)","client":"172.29.81.112:35778","starttimems":1315911888580,"queuetimems":13713,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:04,931 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16338,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@10167387)","client":"172.29.81.101:34867","starttimems":1315911888592,"queuetimems":13652,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:04,958 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16375,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2c9afd4a)","client":"172.29.81.118:55585","starttimems":1315911888582,"queuetimems":13698,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:04,959 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16373,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@736b1c26)","client":"172.29.81.108:45693","starttimems":1315911888585,"queuetimems":13660,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:04,959 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16356,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7d053dfc)","client":"172.29.81.100:41096","starttimems":1315911888602,"queuetimems":13657,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,207 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16601,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@77d84020)","client":"172.29.81.113:47805","starttimems":1315911888605,"queuetimems":13587,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,213 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16507,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1a13857d)","client":"172.29.81.98:45047","starttimems":1315911888705,"queuetimems":13490,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,218 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16632,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@70cc7d13)","client":"172.29.81.99:52426","starttimems":1315911888585,"queuetimems":13669,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,237 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16614,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3c4218e3)","client":"172.29.81.120:43408","starttimems":1315911888622,"queuetimems":13520,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,241 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16618,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@398bf910)","client":"172.29.81.107:50949","starttimems":1315911888622,"queuetimems":13573,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,249 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16646,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@52d7a086)","client":"172.29.81.117:46535","starttimems":1315911888602,"queuetimems":13593,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,249 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16190,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4a6be7e7)","client":"172.29.81.95:51708","starttimems":1315911889058,"queuetimems":13730,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,258 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.4m for region usertable,user1274566160,1315898727282.8c36971138a4bfebe5aeb94c163f2a81. in 10385ms, sequenceid=1663335, compaction requested=true 2011-09-13 04:05:05,260 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16621,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@546f1f81)","client":"172.29.81.114:37636","starttimems":1315911888638,"queuetimems":13493,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,296 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16600,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@56033e42)","client":"172.29.81.94:40535","starttimems":1315911888696,"queuetimems":13496,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,312 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16592,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@bd62dd5)","client":"172.29.81.116:55436","starttimems":1315911888719,"queuetimems":13503,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,312 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16267,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1a5b0105)","client":"172.29.81.102:34147","starttimems":1315911889044,"queuetimems":13760,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,323 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16180,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2b17d84)","client":"172.29.81.109:33058","starttimems":1315911889142,"queuetimems":13547,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,326 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":15834,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@213b6f81)","client":"172.29.81.92:59103","starttimems":1315911889491,"queuetimems":13824,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,326 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16583,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@45158f32)","client":"172.29.81.96:37348","starttimems":1315911888742,"queuetimems":13523,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,331 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16211,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@53afbec2)","client":"172.29.81.106:49354","starttimems":1315911889119,"queuetimems":13571,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,352 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16272,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7bfa49fc)","client":"172.29.81.98:45047","starttimems":1315911889079,"queuetimems":13729,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,387 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":15748,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1c825599)","client":"172.29.81.120:43408","starttimems":1315911889638,"queuetimems":12642,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,450 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13804,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7f4a97ff)","client":"172.29.81.112:35778","starttimems":1315911891645,"queuetimems":14205,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,452 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":14373,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7a4127a2)","client":"172.29.81.107:50949","starttimems":1315911891078,"queuetimems":13827,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,453 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16383,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4b913b62)","client":"172.29.81.96:37348","starttimems":1315911889068,"queuetimems":13739,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,455 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":15839,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2bd1c5b0)","client":"172.29.81.119:44904","starttimems":1315911889615,"queuetimems":12708,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,455 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":15463,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5909bc49)","client":"172.29.81.103:34562","starttimems":1315911889991,"queuetimems":12859,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,456 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":15967,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@e288a4a)","client":"172.29.81.115:58696","starttimems":1315911889488,"queuetimems":13827,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,668 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16052,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@bf810b9)","client":"172.29.81.96:37348","starttimems":1315911889615,"queuetimems":12706,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,701 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13826,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2e4fb4b3)","client":"172.29.81.114:37636","starttimems":1315911891874,"queuetimems":13554,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,705 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13830,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3debabbf)","client":"172.29.81.101:34867","starttimems":1315911891874,"queuetimems":13519,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,708 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":14631,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@25ff0039)","client":"172.29.81.99:52426","starttimems":1315911891076,"queuetimems":13890,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,708 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16035,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@528804c)","client":"172.29.81.110:54126","starttimems":1315911889672,"queuetimems":12600,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,710 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":14616,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1b10877a)","client":"172.29.81.98:45047","starttimems":1315911891093,"queuetimems":13716,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,717 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":14031,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4310cbfb)","client":"172.29.81.118:55585","starttimems":1315911891685,"queuetimems":13368,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,755 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12456,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3587c4af)","client":"172.29.81.95:51708","starttimems":1315911893298,"queuetimems":13819,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,758 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13368,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@73c6b4e7)","client":"172.29.81.113:47805","starttimems":1315911892389,"queuetimems":13103,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,758 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12419,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@79c6f03e)","client":"172.29.81.108:45693","starttimems":1315911893338,"queuetimems":13746,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,759 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":14029,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3b613f9a)","client":"172.29.81.104:50755","starttimems":1315911891729,"queuetimems":13412,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,759 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12459,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@41814ddb)","client":"172.29.81.110:54126","starttimems":1315911893299,"queuetimems":13718,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,763 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12470,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1ecb1c26)","client":"172.29.81.96:37348","starttimems":1315911893292,"queuetimems":13866,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,763 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13316,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@28a51295)","client":"172.29.81.118:55585","starttimems":1315911892446,"queuetimems":13112,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,764 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13324,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@782e94a)","client":"172.29.81.99:52426","starttimems":1315911892439,"queuetimems":13129,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,765 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13580,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3868d465)","client":"172.29.81.96:37348","starttimems":1315911892184,"queuetimems":13330,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,767 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13377,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@526fde68)","client":"172.29.81.109:33058","starttimems":1315911892389,"queuetimems":13144,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,769 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13351,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@67aa2a8a)","client":"172.29.81.98:45047","starttimems":1315911892417,"queuetimems":13119,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,771 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11787,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@61d41576)","client":"172.29.81.115:58696","starttimems":1315911893983,"queuetimems":14349,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,773 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13355,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@626e209)","client":"172.29.81.117:46535","starttimems":1315911892417,"queuetimems":13126,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,773 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13583,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@13198648)","client":"172.29.81.108:45693","starttimems":1315911892189,"queuetimems":13040,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,775 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12476,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@346b16d0)","client":"172.29.81.101:34867","starttimems":1315911893298,"queuetimems":13807,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,777 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13663,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@62a6cc92)","client":"172.29.81.105:44566","starttimems":1315911892113,"queuetimems":13724,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,777 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13592,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@34c0b7d7)","client":"172.29.81.102:34147","starttimems":1315911892184,"queuetimems":13789,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,781 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13803,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@c42b26c)","client":"172.29.81.116:55436","starttimems":1315911891977,"queuetimems":13619,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,782 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12496,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4749e60b)","client":"172.29.81.114:37636","starttimems":1315911893285,"queuetimems":13920,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,783 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12486,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@61429b7c)","client":"172.29.81.103:34562","starttimems":1315911893296,"queuetimems":13860,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,788 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12491,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2fbffe48)","client":"172.29.81.97:46852","starttimems":1315911893296,"queuetimems":13842,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,790 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12450,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@439a110)","client":"172.29.81.100:41096","starttimems":1315911893338,"queuetimems":13744,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,791 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11026,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@6741437f)","client":"172.29.81.113:47805","starttimems":1315911894764,"queuetimems":15082,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,795 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11683,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@22d7f019)","client":"172.29.81.96:37348","starttimems":1315911894111,"queuetimems":14443,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,806 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10975,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5450a50d)","client":"172.29.81.118:55585","starttimems":1315911894830,"queuetimems":15113,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,814 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13134,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@34f8dc84)","client":"172.29.81.106:49354","starttimems":1315911892679,"queuetimems":13324,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,818 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10927,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@74e5f591)","client":"172.29.81.110:54126","starttimems":1315911894889,"queuetimems":15128,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,826 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10586,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@41dc6c65)","client":"172.29.81.95:51708","starttimems":1315911895239,"queuetimems":15348,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:05,826 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10595,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7aa290fb)","client":"172.29.81.96:37348","starttimems":1315911895230,"queuetimems":15400,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:08,656 WARN org.apache.hadoop.hdfs.DFSClient: Error while syncing java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1041) 2011-09-13 04:05:08,657 FATAL org.apache.hadoop.hbase.regionserver.wal.HLog: Could not sync. Requesting close of hlog java.io.IOException: Reflection at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:194) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1041) Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) ... 3 more Caused by: java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) ... 7 more 2011-09-13 04:05:08,657 ERROR org.apache.hadoop.hbase.regionserver.wal.HLog: Error while syncing, requesting close of hlog java.io.IOException: Reflection at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:194) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1041) Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) ... 3 more Caused by: java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) ... 7 more 2011-09-13 04:05:08,659 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315911880125, entries=4038, filesize=68822450. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315911905455 2011-09-13 04:05:08,659 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315911565360 whose highest sequenceid is 1639492 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315911565360 2011-09-13 04:05:08,671 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13700,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@77c36c8c)","client":"172.29.81.119:44904","starttimems":1315911894970,"queuetimems":15161,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:08,673 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12718,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3f0b98c1)","client":"172.29.81.99:52426","starttimems":1315911895953,"queuetimems":16046,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:11,409 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":14281,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1cc5b4e9)","client":"172.29.81.94:40535","starttimems":1315911897127,"queuetimems":17162,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:11,535 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13828,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@44ce24f)","client":"172.29.81.97:46852","starttimems":1315911897706,"queuetimems":17703,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:13,524 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":15810,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1979c47)","client":"172.29.81.98:45047","starttimems":1315911897712,"queuetimems":17666,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:15,392 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/.tmp/7111757769519140114 to hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/family/9212383076246268011 2011-09-13 04:05:15,492 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/family/9212383076246268011, entries=205510, sequenceid=1665082, memsize=64.1m, filesize=28.4m 2011-09-13 04:05:15,830 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16766,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@e825dfe)","client":"172.29.81.97:46852","starttimems":1315911899063,"queuetimems":17154,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:15,917 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16866,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7cd59085)","client":"172.29.81.101:34867","starttimems":1315911899050,"queuetimems":17149,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:15,917 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":17106,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@29034bca)","client":"172.29.81.98:45047","starttimems":1315911898810,"queuetimems":18709,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:16,102 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":17079,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2bd42b0e)","client":"172.29.81.109:33058","starttimems":1315911899022,"queuetimems":17900,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:16,128 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.1m for region usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d. in 3808ms, sequenceid=1665082, compaction requested=false 2011-09-13 04:05:16,174 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":17146,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@429d18af)","client":"172.29.81.96:37348","starttimems":1315911899027,"queuetimems":17135,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:16,285 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16489,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7a32e3f2)","client":"172.29.81.110:54126","starttimems":1315911899794,"queuetimems":17793,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:16,285 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":17206,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4dec4caa)","client":"172.29.81.113:47805","starttimems":1315911899077,"queuetimems":17125,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:16,286 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":17276,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7ada413a)","client":"172.29.81.114:37636","starttimems":1315911899009,"queuetimems":18813,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:16,370 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16496,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@21a89aa9)","client":"172.29.81.118:55585","starttimems":1315911899874,"queuetimems":17872,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:16,409 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":17342,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@22784a4)","client":"172.29.81.108:45693","starttimems":1315911899066,"queuetimems":17121,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:16,457 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":17368,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@45880219)","client":"172.29.81.98:45047","starttimems":1315911899088,"queuetimems":17113,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:16,497 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16605,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@39f645aa)","client":"172.29.81.97:46852","starttimems":1315911899891,"queuetimems":17847,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:16,515 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16828,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2af350c5)","client":"172.29.81.92:59103","starttimems":1315911899686,"queuetimems":17701,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:16,520 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":16866,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1ee2c8ab)","client":"172.29.81.95:51708","starttimems":1315911899653,"queuetimems":17678,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:16,725 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":14244,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1e5ffc3f)","client":"172.29.81.114:37636","starttimems":1315911902480,"queuetimems":20413,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:16,738 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13598,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@22d95cea)","client":"172.29.81.109:33058","starttimems":1315911903139,"queuetimems":20949,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:16,755 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13710,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@be252f4)","client":"172.29.81.97:46852","starttimems":1315911903044,"queuetimems":20964,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:16,755 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13644,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@37b41744)","client":"172.29.81.107:50949","starttimems":1315911903110,"queuetimems":20970,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:16,976 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13898,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2fe2e4d9)","client":"172.29.81.103:34562","starttimems":1315911903077,"queuetimems":20969,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:16,980 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13870,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2463aaa1)","client":"172.29.81.114:37636","starttimems":1315911903110,"queuetimems":20971,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:16,993 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13305,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@51477b6c)","client":"172.29.81.99:52426","starttimems":1315911903687,"queuetimems":21496,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:17,023 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":14008,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@65a39eb4)","client":"172.29.81.119:44904","starttimems":1315911903014,"queuetimems":20941,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:17,247 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13228,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5689e0af)","client":"172.29.81.101:34867","starttimems":1315911904018,"queuetimems":21561,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:17,362 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12120,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@12824975)","client":"172.29.81.110:54126","starttimems":1315911905241,"queuetimems":22652,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:17,367 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13646,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@58376826)","client":"172.29.81.98:45047","starttimems":1315911903720,"queuetimems":21522,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:17,368 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13349,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@15203a0c)","client":"172.29.81.107:50949","starttimems":1315911904018,"queuetimems":21618,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:17,369 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12983,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@794cc028)","client":"172.29.81.99:52426","starttimems":1315911904385,"queuetimems":21902,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:17,379 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12695,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@37df9d75)","client":"172.29.81.114:37636","starttimems":1315911904683,"queuetimems":22172,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:17,389 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12429,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@42e5acf6)","client":"172.29.81.95:51708","starttimems":1315911904959,"queuetimems":22404,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:17,390 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12151,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@74be6cc9)","client":"172.29.81.119:44904","starttimems":1315911905237,"queuetimems":22652,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:17,393 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12486,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@626699dd)","client":"172.29.81.95:51708","starttimems":1315911904906,"queuetimems":22393,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:17,393 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12709,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3dffaea3)","client":"172.29.81.102:34147","starttimems":1315911904683,"queuetimems":22180,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:17,400 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12440,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4013910)","client":"172.29.81.92:59103","starttimems":1315911904959,"queuetimems":22401,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:17,669 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13394,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5fc9281d)","client":"172.29.81.118:55585","starttimems":1315911904274,"queuetimems":21815,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:17,669 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13282,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@f68b90c)","client":"172.29.81.92:59103","starttimems":1315911904386,"queuetimems":21888,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:17,690 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12439,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7b3d44f7)","client":"172.29.81.101:34867","starttimems":1315911905250,"queuetimems":22629,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:17,691 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12483,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@737af356)","client":"172.29.81.114:37636","starttimems":1315911905207,"queuetimems":22645,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:17,692 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12478,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@6af5351c)","client":"172.29.81.110:54126","starttimems":1315911905213,"queuetimems":22646,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:17,709 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12776,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@70d0d081)","client":"172.29.81.113:47805","starttimems":1315911904932,"queuetimems":22394,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:17,724 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12765,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@67770d47)","client":"172.29.81.102:34147","starttimems":1315911904958,"queuetimems":22413,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:17,726 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12794,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@42a8ee04)","client":"172.29.81.118:55585","starttimems":1315911904931,"queuetimems":22415,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:17,737 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12518,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@35f9e5c8)","client":"172.29.81.98:45047","starttimems":1315911905218,"queuetimems":22651,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:18,064 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12813,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2a48e3c3)","client":"172.29.81.101:34867","starttimems":1315911905250,"queuetimems":22659,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:18,064 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12675,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@33999dd4)","client":"172.29.81.103:34562","starttimems":1315911905388,"queuetimems":22655,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:18,067 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12616,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@6cca7a60)","client":"172.29.81.107:50949","starttimems":1315911905450,"queuetimems":22688,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:18,271 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12958,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@74b5aa6)","client":"172.29.81.119:44904","starttimems":1315911905312,"queuetimems":22664,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:18,338 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12885,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@278d0d84)","client":"172.29.81.118:55585","starttimems":1315911905452,"queuetimems":22677,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:18,351 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13023,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4d75f04b)","client":"172.29.81.116:55436","starttimems":1315911905327,"queuetimems":22639,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:18,360 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12906,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@384960f1)","client":"172.29.81.107:50949","starttimems":1315911905453,"queuetimems":22659,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:18,388 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13090,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@54ffdb4b)","client":"172.29.81.120:43408","starttimems":1315911905297,"queuetimems":22653,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:18,405 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13052,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2253e964)","client":"172.29.81.108:45693","starttimems":1315911905352,"queuetimems":22621,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:18,484 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13223,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@122b3c5f)","client":"172.29.81.107:50949","starttimems":1315911905260,"queuetimems":22632,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:18,484 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13170,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@54888a4a)","client":"172.29.81.109:33058","starttimems":1315911905313,"queuetimems":22653,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:18,497 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13165,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@355cc97)","client":"172.29.81.103:34562","starttimems":1315911905331,"queuetimems":22610,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:18,518 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13194,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3954ed2)","client":"172.29.81.108:45693","starttimems":1315911905323,"queuetimems":22637,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:18,534 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13206,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2a0dab3)","client":"172.29.81.107:50949","starttimems":1315911905327,"queuetimems":22611,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:18,637 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12925,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@183cde06)","client":"172.29.81.108:45693","starttimems":1315911905711,"queuetimems":22172,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:18,873 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13204,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5a468504)","client":"172.29.81.97:46852","starttimems":1315911905668,"queuetimems":22714,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:18,873 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13417,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4bc6f7e9)","client":"172.29.81.107:50949","starttimems":1315911905455,"queuetimems":22640,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:18,874 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13166,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4137dee1)","client":"172.29.81.97:46852","starttimems":1315911905708,"queuetimems":22324,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:18,889 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13171,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@58a5d87e)","client":"172.29.81.98:45047","starttimems":1315911905717,"queuetimems":21759,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:18,889 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13183,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@36dad444)","client":"172.29.81.97:46852","starttimems":1315911905705,"queuetimems":22377,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:18,891 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13181,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5716c1fd)","client":"172.29.81.97:46852","starttimems":1315911905709,"queuetimems":22281,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:18,898 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13442,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@29528b93)","client":"172.29.81.103:34562","starttimems":1315911905455,"queuetimems":22576,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:18,933 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13476,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@6cf7dfa4)","client":"172.29.81.113:47805","starttimems":1315911905456,"queuetimems":22555,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:19,152 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13381,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@310ccc23)","client":"172.29.81.97:46852","starttimems":1315911905770,"queuetimems":21462,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:19,152 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13386,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2b3cfd1a)","client":"172.29.81.103:34562","starttimems":1315911905765,"queuetimems":21470,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:19,152 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13379,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@6f01cc72)","client":"172.29.81.114:37636","starttimems":1315911905773,"queuetimems":21412,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:19,153 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13394,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3f2e26fd)","client":"172.29.81.97:46852","starttimems":1315911905758,"queuetimems":21594,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:19,155 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13452,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7418871)","client":"172.29.81.97:46852","starttimems":1315911905702,"queuetimems":22428,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:19,156 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13384,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5f64b66f)","client":"172.29.81.120:43408","starttimems":1315911905771,"queuetimems":21456,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:19,156 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13396,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4d056256)","client":"172.29.81.109:33058","starttimems":1315911905759,"queuetimems":21517,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:05:20,252 WARN org.apache.hadoop.hbase.regionserver.wal.HLog: IPC Server handler 37 on 60020 took 1095 ms appending an edit to hlog; editcount=1260, len~=17.0k -- 2011-09-13 04:58:43,995 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12136,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@ee0ba79)","client":"172.29.81.93:59543","starttimems":1315915111857,"queuetimems":8366,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:58:43,996 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12124,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5ee9f3c9)","client":"172.29.81.92:45170","starttimems":1315915111871,"queuetimems":8380,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:58:44,093 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11551,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@17307b37)","client":"172.29.81.111:50796","starttimems":1315915112541,"queuetimems":9026,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:58:44,334 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11514,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5594a5d4)","client":"172.29.81.109:41078","starttimems":1315915112819,"queuetimems":9297,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:58:44,392 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10549,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@359fe202)","client":"172.29.81.105:33388","starttimems":1315915113842,"queuetimems":10231,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:58:45,517 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12668,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3c588844)","client":"172.29.81.118:56786","starttimems":1315915112848,"queuetimems":9265,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:58:45,908 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12196,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@33f7c0d0)","client":"172.29.81.111:50796","starttimems":1315915113711,"queuetimems":10114,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:58:46,293 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12113,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4bc0e0ba)","client":"172.29.81.118:56786","starttimems":1315915114179,"queuetimems":10514,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:58:46,535 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12448,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2714a685)","client":"172.29.81.106:41992","starttimems":1315915114086,"queuetimems":10456,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:58:46,536 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10128,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3cc079ce)","client":"172.29.81.118:56786","starttimems":1315915116407,"queuetimems":12670,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:58:46,542 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10109,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@6fd13ed)","client":"172.29.81.112:50254","starttimems":1315915116433,"queuetimems":12681,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:58:46,553 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12420,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3dd8b2d6)","client":"172.29.81.95:54100","starttimems":1315915114132,"queuetimems":10482,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:58:46,572 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12439,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@41dfd85a)","client":"172.29.81.112:50254","starttimems":1315915114132,"queuetimems":10496,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:58:46,601 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12414,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@a4e026)","client":"172.29.81.109:41078","starttimems":1315915114186,"queuetimems":10500,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:58:47,372 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10691,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3bd46027)","client":"172.29.81.118:56786","starttimems":1315915116680,"queuetimems":12876,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:58:47,428 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10617,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@73392818)","client":"172.29.81.115:37386","starttimems":1315915116810,"queuetimems":12865,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:58:47,430 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10651,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@131293e1)","client":"172.29.81.112:50254","starttimems":1315915116779,"queuetimems":12898,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:58:47,497 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10296,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2ac987e1)","client":"172.29.81.118:56786","starttimems":1315915117199,"queuetimems":13225,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:58:47,501 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":10366,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4e4ae589)","client":"172.29.81.118:56786","starttimems":1315915117134,"queuetimems":13173,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 04:58:51,777 WARN org.apache.hadoop.hbase.regionserver.wal.HLog: IPC Server handler 71 on 60020 took 1083 ms appending an edit to hlog; editcount=3714, len~=15.7k 2011-09-13 04:58:52,298 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315915101960, entries=3783, filesize=64496683. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315915131816 2011-09-13 04:58:52,301 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315914816829 whose highest sequenceid is 1945238 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315914816829 2011-09-13 04:58:52,341 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315914858085 whose highest sequenceid is 1950593 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315914858085 2011-09-13 04:58:55,284 WARN org.apache.hadoop.hbase.regionserver.wal.HLog: IPC Server handler 71 on 60020 took 1201 ms appending an edit to hlog; editcount=850, len~=9.9k 2011-09-13 04:59:01,047 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 3 file(s) in family of usertable,user224923440,1315898727290.766e5174f1140d766dd0af91cbd1d845.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/766e5174f1140d766dd0af91cbd1d845/family/2091510586236599598, size=88.3m; total size for store is 2.1g 2011-09-13 04:59:01,047 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user224923440,1315898727290.766e5174f1140d766dd0af91cbd1d845., storeName=family, fileCount=3, fileSize=88.3m, priority=-1, time=3929411505722297; duration=30sec 2011-09-13 04:59:03,612 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/.tmp/5844369133498722362 to hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/family/3082365981510309195 2011-09-13 04:59:03,653 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/family/3082365981510309195, entries=205940, sequenceid=1976176, memsize=64.3m, filesize=28.5m 2011-09-13 04:59:03,659 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.3m for region usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd. in 7284ms, sequenceid=1976176, compaction requested=true 2011-09-13 04:59:03,659 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd. 2011-09-13 04:59:03,660 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 3 file(s) in family of usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd. into hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/.tmp, seqid=1976176, totalSize=94.8m 2011-09-13 04:59:03,732 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315915131816, entries=4491, filesize=76318843. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315915143659 2011-09-13 04:59:06,050 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/50d20bc0e631f85cd8b961fcf983914a/.tmp/9077885320704696950 to hdfs://c0309.xxxxx:17020/hbase/usertable/50d20bc0e631f85cd8b961fcf983914a/family/7391594536983437860 2011-09-13 04:59:06,138 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/50d20bc0e631f85cd8b961fcf983914a/family/7391594536983437860, entries=225300, sequenceid=1979065, memsize=70.5m, filesize=31.4m 2011-09-13 04:59:07,707 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~70.5m for region usertable,user1526266200,1315898727285.50d20bc0e631f85cd8b961fcf983914a. in 4048ms, sequenceid=1979065, compaction requested=false 2011-09-13 04:59:18,998 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 3 file(s) in family of usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/family/3408270660049743912, size=94.8m; total size for store is 2.2g 2011-09-13 04:59:18,998 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd., storeName=family, fileCount=3, fileSize=94.8m, priority=2, time=3929444916687297; duration=15sec 2011-09-13 04:59:22,704 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/.tmp/5612012081204507576 to hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/family/1248389846731709586 2011-09-13 04:59:22,740 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/family/1248389846731709586, entries=212700, sequenceid=1979540, memsize=66.5m, filesize=29.6m 2011-09-13 04:59:23,932 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~66.5m for region usertable,user1772610920,1315898727287.77b5b2f51884554ebf951146d25f1bd2. in 16225ms, sequenceid=1979540, compaction requested=false 2011-09-13 04:59:25,828 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315915143659, entries=4238, filesize=71851028. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315915163933 2011-09-13 04:59:25,829 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315914876737 whose highest sequenceid is 1957397 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315914876737 2011-09-13 04:59:27,905 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/.tmp/2537081895108052757 to hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/family/4119578581922142894 2011-09-13 04:59:27,967 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/family/4119578581922142894, entries=209960, sequenceid=1983401, memsize=65.7m, filesize=29.2m 2011-09-13 04:59:28,860 WARN org.apache.hadoop.hbase.regionserver.wal.HLog: IPC Server handler 46 on 60020 took 1157 ms appending an edit to hlog; editcount=1288, len~=25.5k 2011-09-13 04:59:29,044 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~65.7m for region usertable,user1649438560,1315898727286.29ec66424227dc3cbe3d8b2311764fc9. in 5111ms, sequenceid=1983401, compaction requested=false 2011-09-13 04:59:34,186 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315915163933, entries=3807, filesize=64818149. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315915171815 2011-09-13 04:59:34,187 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315914919672 whose highest sequenceid is 1962598 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315914919672 2011-09-13 04:59:43,987 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/.tmp/8585665761863928976 to hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/family/2342198596905617732 2011-09-13 04:59:44,041 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/family/2342198596905617732, entries=206330, sequenceid=1987899, memsize=64.5m, filesize=28.7m 2011-09-13 04:59:44,555 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.5m for region usertable,user1022866120,1315898727279.f3cb8bb14102b43abbb2b45cd2fc4945. in 8165ms, sequenceid=1987899, compaction requested=true 2011-09-13 04:59:51,074 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/.tmp/5586599034068468642 to hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/family/4413663536653297907 2011-09-13 04:59:51,131 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/family/4413663536653297907, entries=212580, sequenceid=1990559, memsize=66.5m, filesize=29.6m 2011-09-13 04:59:51,133 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~66.5m for region usertable,user2029666280,1315898727289.8cb227d0fbbb2ad66e2f3d25cdfa0f89. in 6578ms, sequenceid=1990559, compaction requested=false 2011-09-13 04:59:52,530 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315915171815, entries=4695, filesize=79706815. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315915191133 2011-09-13 04:59:52,532 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315914940681 whose highest sequenceid is 1966608 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315914940681 2011-09-13 04:59:56,464 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/.tmp/8073257242463641405 to hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/family/7265323381461409806 2011-09-13 04:59:56,547 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/family/7265323381461409806, entries=208980, sequenceid=1991805, memsize=65.2m, filesize=28.9m 2011-09-13 04:59:56,554 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~65.2m for region usertable,user728323520,1315898727295.a623bd664ea942437bf6ae1073fb5560. in 5421ms, sequenceid=1991805, compaction requested=true 2011-09-13 05:00:52,159 WARN org.apache.hadoop.hbase.regionserver.MemStoreFlusher: Region usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d. has too many store files; delaying flush up to 90000ms 2011-09-13 05:01:04,868 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/169b91a09b9b10a23602aaea5e8b9fc2/.tmp/8318377558958490724 to hdfs://c0309.xxxxx:17020/hbase/usertable/169b91a09b9b10a23602aaea5e8b9fc2/family/8995369354784308690 2011-09-13 05:01:05,151 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/169b91a09b9b10a23602aaea5e8b9fc2/family/8995369354784308690, entries=204880, sequenceid=1992105, memsize=64.1m, filesize=28.5m 2011-09-13 05:01:05,436 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.1m for region usertable,user1901138600,1315898727288.169b91a09b9b10a23602aaea5e8b9fc2. in 8874ms, sequenceid=1992105, compaction requested=false 2011-09-13 05:01:10,140 WARN org.apache.hadoop.hbase.regionserver.wal.HLog: IPC Server handler 45 on 60020 took 1048 ms appending an edit to hlog; editcount=3231, len~=17.0k 2011-09-13 05:01:12,445 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/.tmp/6577149898440817331 to hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/family/2047235477360768270 2011-09-13 05:01:12,522 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/family/2047235477360768270, entries=220200, sequenceid=1994392, memsize=68.7m, filesize=30.5m 2011-09-13 05:01:12,881 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~68.7m for region usertable,user980023560,1315898727298.a4521b468fb6ac94dd495ee2db515ae0. in 7444ms, sequenceid=1994392, compaction requested=true 2011-09-13 05:01:12,882 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user980023560,1315898727298.a4521b468fb6ac94dd495ee2db515ae0. 2011-09-13 05:01:12,882 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 3 file(s) in family of usertable,user980023560,1315898727298.a4521b468fb6ac94dd495ee2db515ae0. into hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/.tmp, seqid=1994392, totalSize=88.1m 2011-09-13 05:01:19,647 WARN org.apache.hadoop.hdfs.DFSClient: Error while syncing java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1041) 2011-09-13 05:01:19,648 FATAL org.apache.hadoop.hbase.regionserver.wal.HLog: Could not sync. Requesting close of hlog java.io.IOException: Reflection at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:194) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1041) Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) ... 3 more Caused by: java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) ... 7 more 2011-09-13 05:01:19,648 ERROR org.apache.hadoop.hbase.regionserver.wal.HLog: Error while syncing, requesting close of hlog java.io.IOException: Reflection at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:194) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1041) Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) ... 3 more Caused by: java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) ... 7 more 2011-09-13 05:01:19,651 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315915191133, entries=3978, filesize=67424726. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315915272881 2011-09-13 05:01:19,658 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13398,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@202b5d01)","client":"172.29.81.99:56382","starttimems":1315915266259,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:19,693 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13470,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@319e447b)","client":"172.29.81.99:56382","starttimems":1315915266221,"queuetimems":1,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:19,693 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13411,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@da200b0)","client":"172.29.81.100:34711","starttimems":1315915266281,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:19,695 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13298,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@6ee5e5b9)","client":"172.29.81.104:58582","starttimems":1315915266396,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:19,701 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13238,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1fd93d1c)","client":"172.29.81.104:58582","starttimems":1315915266462,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:19,720 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13331,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@27b8cd42)","client":"172.29.81.102:44548","starttimems":1315915266388,"queuetimems":1,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:19,728 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13210,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@273379ca)","client":"172.29.81.100:34711","starttimems":1315915266517,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:19,729 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13343,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3d7ad019)","client":"172.29.81.103:43239","starttimems":1315915266385,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:19,745 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13369,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@67f6f824)","client":"172.29.81.111:50938","starttimems":1315915266375,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:20,045 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13398,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7af05f)","client":"172.29.81.92:55586","starttimems":1315915266646,"queuetimems":1,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:20,085 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13649,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@25ae9e92)","client":"172.29.81.101:41690","starttimems":1315915266436,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:20,261 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":13634,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@c33e76a)","client":"172.29.81.114:45479","starttimems":1315915266626,"queuetimems":0,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:20,760 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12091,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@76e14f77)","client":"172.29.81.111:50938","starttimems":1315915268668,"queuetimems":2015,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:21,894 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11650,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3524e158)","client":"172.29.81.101:41690","starttimems":1315915270205,"queuetimems":3548,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:22,997 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12552,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@39d3e30b)","client":"172.29.81.107:55330","starttimems":1315915270444,"queuetimems":3750,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,034 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11915,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@79ae46f1)","client":"172.29.81.102:44548","starttimems":1315915271118,"queuetimems":4399,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,345 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11889,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4ab46275)","client":"172.29.81.118:42419","starttimems":1315915271456,"queuetimems":4506,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,356 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12240,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@126d2b17)","client":"172.29.81.105:42899","starttimems":1315915271115,"queuetimems":4405,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,364 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12163,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1c776c78)","client":"172.29.81.104:58582","starttimems":1315915271200,"queuetimems":4301,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,380 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12261,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4d5b19ac)","client":"172.29.81.113:55723","starttimems":1315915271118,"queuetimems":4401,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,388 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12255,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@409f7065)","client":"172.29.81.100:34711","starttimems":1315915271132,"queuetimems":4351,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,388 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12273,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@427dc773)","client":"172.29.81.113:55723","starttimems":1315915271114,"queuetimems":4413,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,388 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12243,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@238c1808)","client":"172.29.81.100:34711","starttimems":1315915271144,"queuetimems":4356,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,407 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11974,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3f79ef1a)","client":"172.29.81.93:59821","starttimems":1315915271432,"queuetimems":4502,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,407 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12234,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@168dae69)","client":"172.29.81.100:34711","starttimems":1315915271172,"queuetimems":4355,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,408 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12255,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2fb57b10)","client":"172.29.81.100:34711","starttimems":1315915271152,"queuetimems":4351,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,620 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12163,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@6e95aba)","client":"172.29.81.113:55723","starttimems":1315915271456,"queuetimems":4502,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,621 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12474,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@42436ce1)","client":"172.29.81.100:34711","starttimems":1315915271146,"queuetimems":4352,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,622 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12432,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@44f1ceaa)","client":"172.29.81.111:50938","starttimems":1315915271189,"queuetimems":4317,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,623 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12432,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1ba3afe4)","client":"172.29.81.92:55586","starttimems":1315915271190,"queuetimems":4318,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,625 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12497,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@38cbb587)","client":"172.29.81.111:50938","starttimems":1315915271127,"queuetimems":4376,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,625 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12150,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@672cf604)","client":"172.29.81.103:43239","starttimems":1315915271474,"queuetimems":4518,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,634 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11979,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@135e10ff)","client":"172.29.81.98:53877","starttimems":1315915271654,"queuetimems":4565,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,634 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12449,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@4addf48)","client":"172.29.81.107:55330","starttimems":1315915271184,"queuetimems":4354,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,688 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12546,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@40265478)","client":"172.29.81.107:55330","starttimems":1315915271141,"queuetimems":4359,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,689 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12052,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1473b32f)","client":"172.29.81.112:34665","starttimems":1315915271636,"queuetimems":4549,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,690 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12537,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@37f8dd26)","client":"172.29.81.100:34711","starttimems":1315915271152,"queuetimems":4346,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,690 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12518,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@337f067f)","client":"172.29.81.100:34711","starttimems":1315915271171,"queuetimems":4360,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,690 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12517,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@58839642)","client":"172.29.81.100:34711","starttimems":1315915271172,"queuetimems":4349,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,706 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12069,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@34da1e96)","client":"172.29.81.111:50938","starttimems":1315915271636,"queuetimems":4577,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,760 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12167,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@612d9dc4)","client":"172.29.81.113:55723","starttimems":1315915271592,"queuetimems":4590,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,774 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12119,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7d6630ff)","client":"172.29.81.107:55330","starttimems":1315915271654,"queuetimems":4390,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,780 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11524,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@22fe7f10)","client":"172.29.81.111:50938","starttimems":1315915272256,"queuetimems":4713,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,835 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12231,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@746bf96)","client":"172.29.81.104:58582","starttimems":1315915271603,"queuetimems":4595,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,835 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12206,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@35cc1495)","client":"172.29.81.105:42899","starttimems":1315915271628,"queuetimems":4600,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,835 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12131,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@76fad027)","client":"172.29.81.108:41041","starttimems":1315915271704,"queuetimems":4364,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,837 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11581,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7f0ceab0)","client":"172.29.81.101:41690","starttimems":1315915272256,"queuetimems":4752,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,837 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12046,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@290409fa)","client":"172.29.81.112:34665","starttimems":1315915271791,"queuetimems":4354,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,859 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12154,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3daca7b3)","client":"172.29.81.105:42899","starttimems":1315915271704,"queuetimems":4361,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,859 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12306,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1235f867)","client":"172.29.81.101:41690","starttimems":1315915271552,"queuetimems":4582,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,862 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12161,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@72085f5c)","client":"172.29.81.107:55330","starttimems":1315915271700,"queuetimems":4428,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,862 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12157,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@3accb81)","client":"172.29.81.102:44548","starttimems":1315915271704,"queuetimems":4317,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,869 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11607,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@725e853b)","client":"172.29.81.93:59821","starttimems":1315915272261,"queuetimems":4634,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,874 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12172,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@76ec3893)","client":"172.29.81.92:55586","starttimems":1315915271702,"queuetimems":4383,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,908 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12307,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@57bca656)","client":"172.29.81.108:41041","starttimems":1315915271600,"queuetimems":4596,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,908 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12202,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@224c851a)","client":"172.29.81.111:50938","starttimems":1315915271705,"queuetimems":4292,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:23,910 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11675,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@7bcae694)","client":"172.29.81.98:53877","starttimems":1315915272234,"queuetimems":4752,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:24,160 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12455,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@257b60eb)","client":"172.29.81.101:41690","starttimems":1315915271705,"queuetimems":4306,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:24,162 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11880,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@767ba2e0)","client":"172.29.81.103:43239","starttimems":1315915272281,"queuetimems":4550,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:24,176 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11909,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@17fa3083)","client":"172.29.81.101:41690","starttimems":1315915272266,"queuetimems":4638,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:24,179 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12474,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@10ce97f0)","client":"172.29.81.103:43239","starttimems":1315915271704,"queuetimems":4319,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:24,180 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12393,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@25bf5d11)","client":"172.29.81.100:34711","starttimems":1315915271786,"queuetimems":4352,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:24,183 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12480,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@62172b25)","client":"172.29.81.107:55330","starttimems":1315915271702,"queuetimems":4389,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:24,185 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11910,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@12f8917)","client":"172.29.81.111:50938","starttimems":1315915272274,"queuetimems":4612,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:24,186 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11923,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@6c3131ce)","client":"172.29.81.98:53877","starttimems":1315915272262,"queuetimems":4635,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:24,186 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11935,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@57d1459c)","client":"172.29.81.113:55723","starttimems":1315915272250,"queuetimems":4748,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:24,189 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12598,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@5eab91ee)","client":"172.29.81.108:41041","starttimems":1315915271590,"queuetimems":4617,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:24,192 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12487,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1ed76d43)","client":"172.29.81.111:50938","starttimems":1315915271704,"queuetimems":4326,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:24,200 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11932,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@185f542b)","client":"172.29.81.103:43239","starttimems":1315915272267,"queuetimems":4634,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:24,202 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11923,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1aca96fc)","client":"172.29.81.104:58582","starttimems":1315915272278,"queuetimems":4573,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:24,206 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11949,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2d3789ec)","client":"172.29.81.107:55330","starttimems":1315915272256,"queuetimems":4629,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:24,206 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12503,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@193ffc25)","client":"172.29.81.115:56184","starttimems":1315915271702,"queuetimems":4426,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:24,245 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12539,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@6985ef5a)","client":"172.29.81.92:55586","starttimems":1315915271704,"queuetimems":4333,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:24,247 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11981,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@21d773e2)","client":"172.29.81.105:42899","starttimems":1315915272266,"queuetimems":4635,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:24,248 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11972,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@47f90bc9)","client":"172.29.81.112:34665","starttimems":1315915272275,"queuetimems":4607,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:24,250 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12548,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@6c2cdf86)","client":"172.29.81.107:55330","starttimems":1315915271701,"queuetimems":4426,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:24,251 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12229,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@71324b00)","client":"172.29.81.114:45479","starttimems":1315915272021,"queuetimems":4566,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:24,253 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12231,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@59d37aaa)","client":"172.29.81.104:58582","starttimems":1315915272021,"queuetimems":4570,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:24,255 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":12005,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@1c790d3b)","client":"172.29.81.98:53877","starttimems":1315915272249,"queuetimems":4748,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:24,278 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11993,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@2f3dfa1)","client":"172.29.81.104:58582","starttimems":1315915272284,"queuetimems":4537,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:24,278 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11986,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@472d464f)","client":"172.29.81.107:55330","starttimems":1315915272291,"queuetimems":4506,"class":"HRegionServer","responsesize":0,"method":"multi"} 2011-09-13 05:01:24,291 WARN org.apache.hadoop.ipc.HBaseServer: (responseTooSlow): {"processingtimems":11869,"call":"multi(org.apache.hadoop.hbase.client.MultiAction@164ad00b)","client":"172.29.81.114:45479","starttimems":1315915272421,"queuetimems":4357,"class":"HRegionServer","responsesize":0,"method":"multi"} -- 2011-09-13 05:32:30,062 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~65.2m for region usertable,user1274566160,1315898727282.8c36971138a4bfebe5aeb94c163f2a81. in 11175ms, sequenceid=2237554, compaction requested=false 2011-09-13 05:32:30,062 WARN org.apache.hadoop.hbase.regionserver.MemStoreFlusher: Region usertable,user2029666280,1315898727289.8cb227d0fbbb2ad66e2f3d25cdfa0f89. has too many store files; delaying flush up to 90000ms 2011-09-13 05:32:30,063 WARN org.apache.hadoop.hbase.regionserver.MemStoreFlusher: Region usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d. has too many store files; delaying flush up to 90000ms 2011-09-13 05:32:31,950 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/169b91a09b9b10a23602aaea5e8b9fc2/.tmp/8933018641573531273 to hdfs://c0309.xxxxx:17020/hbase/usertable/169b91a09b9b10a23602aaea5e8b9fc2/family/6560480393291229649 2011-09-13 05:32:32,016 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/169b91a09b9b10a23602aaea5e8b9fc2/family/6560480393291229649, entries=211870, sequenceid=2239953, memsize=66.3m, filesize=29.5m 2011-09-13 05:32:32,068 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~66.3m for region usertable,user1901138600,1315898727288.169b91a09b9b10a23602aaea5e8b9fc2. in 2005ms, sequenceid=2239953, compaction requested=true 2011-09-13 05:32:32,164 WARN org.apache.hadoop.hbase.regionserver.MemStoreFlusher: Region usertable,user980023560,1315898727298.a4521b468fb6ac94dd495ee2db515ae0. has too many store files; delaying flush up to 90000ms 2011-09-13 05:32:41,684 INFO org.apache.hadoop.hbase.regionserver.Store: Completed major compaction of 5 file(s) in family of usertable,user1274566160,1315898727282.8c36971138a4bfebe5aeb94c163f2a81.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/8c36971138a4bfebe5aeb94c163f2a81/family/3573528311516931615, size=1.9g; total size for store is 2.5g 2011-09-13 05:32:41,684 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user1274566160,1315898727282.8c36971138a4bfebe5aeb94c163f2a81., storeName=family, fileCount=5, fileSize=2.0g, priority=2, time=3927441725569297; duration=5mins, 26sec 2011-09-13 05:32:41,684 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user1526266200,1315898727285.50d20bc0e631f85cd8b961fcf983914a. 2011-09-13 05:32:41,685 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 4 file(s) in family of usertable,user1526266200,1315898727285.50d20bc0e631f85cd8b961fcf983914a. into hdfs://c0309.xxxxx:17020/hbase/usertable/50d20bc0e631f85cd8b961fcf983914a/.tmp, seqid=2210162, totalSize=391.4m 2011-09-13 05:32:44,661 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315917129785, entries=3774, filesize=64279152. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315917164518 2011-09-13 05:32:57,328 INFO org.apache.hadoop.hbase.regionserver.MemStoreFlusher: Waited 90055ms on a compaction to clean up 'too many store files'; waited long enough... proceeding with flush of usertable,user1146038480,1315898727281.f6d7a2cd3251b638e0f60e6e46e71a88. 2011-09-13 05:33:03,577 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/.tmp/5411276420773132704 to hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/family/6752410518231295300 2011-09-13 05:33:03,614 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/family/6752410518231295300, entries=321800, sequenceid=2240402, memsize=100.6m, filesize=44.8m 2011-09-13 05:33:03,615 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~100.6m for region usertable,user1146038480,1315898727281.f6d7a2cd3251b638e0f60e6e46e71a88. in 6286ms, sequenceid=2240402, compaction requested=true 2011-09-13 05:33:36,996 INFO org.apache.hadoop.hbase.regionserver.MemStoreFlusher: Waited 90431ms on a compaction to clean up 'too many store files'; waited long enough... proceeding with flush of usertable,user1526266200,1315898727285.50d20bc0e631f85cd8b961fcf983914a. 2011-09-13 05:33:42,343 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/50d20bc0e631f85cd8b961fcf983914a/.tmp/7720958668017173412 to hdfs://c0309.xxxxx:17020/hbase/usertable/50d20bc0e631f85cd8b961fcf983914a/family/5767477177444407473 2011-09-13 05:33:42,648 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/50d20bc0e631f85cd8b961fcf983914a/family/5767477177444407473, entries=244150, sequenceid=2240403, memsize=76.4m, filesize=34.0m 2011-09-13 05:33:42,649 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~76.4m for region usertable,user1526266200,1315898727285.50d20bc0e631f85cd8b961fcf983914a. in 5652ms, sequenceid=2240403, compaction requested=false 2011-09-13 05:33:42,649 INFO org.apache.hadoop.hbase.regionserver.MemStoreFlusher: Waited 92909ms on a compaction to clean up 'too many store files'; waited long enough... proceeding with flush of usertable,user1772610920,1315898727287.77b5b2f51884554ebf951146d25f1bd2. 2011-09-13 05:33:46,358 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 4 file(s) in family of usertable,user1526266200,1315898727285.50d20bc0e631f85cd8b961fcf983914a.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/50d20bc0e631f85cd8b961fcf983914a/family/4074195823729068622, size=391.4m; total size for store is 2.5g 2011-09-13 05:33:46,358 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user1526266200,1315898727285.50d20bc0e631f85cd8b961fcf983914a., storeName=family, fileCount=4, fileSize=391.4m, priority=-2, time=3931251029894297; duration=1mins, 4sec 2011-09-13 05:33:46,358 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d. 2011-09-13 05:33:46,359 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 4 file(s) in family of usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d. into hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/.tmp, seqid=2212168, totalSize=191.2m 2011-09-13 05:33:49,669 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/.tmp/1834358873578760096 to hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/family/5387982848782185541 2011-09-13 05:33:49,714 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/family/5387982848782185541, entries=237000, sequenceid=2240502, memsize=74.1m, filesize=33.0m 2011-09-13 05:33:49,716 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~74.1m for region usertable,user1772610920,1315898727287.77b5b2f51884554ebf951146d25f1bd2. in 7066ms, sequenceid=2240502, compaction requested=true 2011-09-13 05:33:49,716 INFO org.apache.hadoop.hbase.regionserver.MemStoreFlusher: Waited 91965ms on a compaction to clean up 'too many store files'; waited long enough... proceeding with flush of usertable,user1649438560,1315898727286.29ec66424227dc3cbe3d8b2311764fc9. 2011-09-13 05:33:52,636 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/.tmp/3849904914954819034 to hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/family/5854277849012468897 2011-09-13 05:33:52,670 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/family/5854277849012468897, entries=231810, sequenceid=2240503, memsize=72.5m, filesize=32.3m 2011-09-13 05:33:52,673 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~72.5m for region usertable,user1649438560,1315898727286.29ec66424227dc3cbe3d8b2311764fc9. in 2957ms, sequenceid=2240503, compaction requested=true 2011-09-13 05:33:52,673 INFO org.apache.hadoop.hbase.regionserver.MemStoreFlusher: Waited 92779ms on a compaction to clean up 'too many store files'; waited long enough... proceeding with flush of usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d. 2011-09-13 05:33:53,862 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/.tmp/3599169272447119614 to hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/family/1634269110963527104 2011-09-13 05:33:53,906 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/family/1634269110963527104, entries=227310, sequenceid=2240504, memsize=70.9m, filesize=31.4m 2011-09-13 05:33:53,908 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~70.9m for region usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d. in 1235ms, sequenceid=2240504, compaction requested=false 2011-09-13 05:33:53,908 INFO org.apache.hadoop.hbase.regionserver.MemStoreFlusher: Waited 94546ms on a compaction to clean up 'too many store files'; waited long enough... proceeding with flush of usertable,user2029666280,1315898727289.8cb227d0fbbb2ad66e2f3d25cdfa0f89. 2011-09-13 05:33:57,019 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/.tmp/525961939915500642 to hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/family/693401797749034832 2011-09-13 05:33:57,057 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/family/693401797749034832, entries=230160, sequenceid=2240505, memsize=72.0m, filesize=32.1m 2011-09-13 05:33:57,059 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~72.0m for region usertable,user2029666280,1315898727289.8cb227d0fbbb2ad66e2f3d25cdfa0f89. in 3151ms, sequenceid=2240505, compaction requested=false 2011-09-13 05:33:57,059 INFO org.apache.hadoop.hbase.regionserver.MemStoreFlusher: Waited 99354ms on a compaction to clean up 'too many store files'; waited long enough... proceeding with flush of usertable,user224923440,1315898727290.766e5174f1140d766dd0af91cbd1d845. 2011-09-13 05:33:58,245 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/766e5174f1140d766dd0af91cbd1d845/.tmp/6219034244655255377 to hdfs://c0309.xxxxx:17020/hbase/usertable/766e5174f1140d766dd0af91cbd1d845/family/8963583272857398681 2011-09-13 05:33:58,276 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/766e5174f1140d766dd0af91cbd1d845/family/8963583272857398681, entries=231430, sequenceid=2240506, memsize=72.2m, filesize=32.0m 2011-09-13 05:33:58,278 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~72.2m for region usertable,user224923440,1315898727290.766e5174f1140d766dd0af91cbd1d845. in 1219ms, sequenceid=2240506, compaction requested=true 2011-09-13 05:34:02,788 INFO org.apache.hadoop.hbase.regionserver.MemStoreFlusher: Waited 90624ms on a compaction to clean up 'too many store files'; waited long enough... proceeding with flush of usertable,user980023560,1315898727298.a4521b468fb6ac94dd495ee2db515ae0. 2011-09-13 05:34:04,556 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 4 file(s) in family of usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/family/3943000571174402735, size=191.2m; total size for store is 2.4g 2011-09-13 05:34:04,556 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d., storeName=family, fileCount=4, fileSize=191.2m, priority=-2, time=3931283236170297; duration=18sec 2011-09-13 05:34:04,556 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user224923440,1315898727290.766e5174f1140d766dd0af91cbd1d845. 2011-09-13 05:34:04,557 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 5 file(s) in family of usertable,user224923440,1315898727290.766e5174f1140d766dd0af91cbd1d845. into hdfs://c0309.xxxxx:17020/hbase/usertable/766e5174f1140d766dd0af91cbd1d845/.tmp, seqid=2240506, totalSize=391.6m 2011-09-13 05:34:06,864 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/.tmp/317933499472403831 to hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/family/6880082667090993607 2011-09-13 05:34:06,910 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/family/6880082667090993607, entries=208730, sequenceid=2240507, memsize=65.1m, filesize=28.9m 2011-09-13 05:34:06,912 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~65.1m for region usertable,user980023560,1315898727298.a4521b468fb6ac94dd495ee2db515ae0. in 4124ms, sequenceid=2240507, compaction requested=true 2011-09-13 05:34:22,723 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/.tmp/4470820988310808149 to hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/family/3112020262005003894 2011-09-13 05:34:22,896 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/family/3112020262005003894, entries=205210, sequenceid=2241661, memsize=64.0m, filesize=28.4m 2011-09-13 05:34:23,003 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.0m for region usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd. in 3562ms, sequenceid=2241661, compaction requested=false 2011-09-13 05:34:26,822 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315917164518, entries=3831, filesize=64974477. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315917266677 2011-09-13 05:34:26,824 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315916890429 whose highest sequenceid is 2204167 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315916890429 2011-09-13 05:34:26,842 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315916911339 whose highest sequenceid is 2208303 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315916911339 2011-09-13 05:34:26,875 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315916931217 whose highest sequenceid is 2212166 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315916931217 2011-09-13 05:34:26,893 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315916969424 whose highest sequenceid is 2216149 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315916969424 2011-09-13 05:34:34,472 WARN org.apache.hadoop.hbase.regionserver.wal.HLog: IPC Server handler 21 on 60020 took 1378 ms appending an edit to hlog; editcount=2146, len~=14.2k 2011-09-13 05:34:35,152 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/.tmp/8705549606058488132 to hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/family/2650780585828873424 2011-09-13 05:34:35,234 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/family/2650780585828873424, entries=206830, sequenceid=2245273, memsize=64.5m, filesize=28.6m 2011-09-13 05:34:35,471 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.5m for region usertable,user728323520,1315898727295.a623bd664ea942437bf6ae1073fb5560. in 5377ms, sequenceid=2245273, compaction requested=false 2011-09-13 05:34:41,253 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315917266677, entries=3781, filesize=64564953. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315917280910 2011-09-13 05:34:41,253 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315917051897 whose highest sequenceid is 2220217 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315917051897 2011-09-13 05:34:41,328 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315917062839 whose highest sequenceid is 2224000 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315917062839 2011-09-13 05:34:41,377 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: moving old hlog file /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315917081912 whose highest sequenceid is 2227747 to /hbase/.oldlogs/c0334.xxxxx%2C60020%2C1315897876046.1315917081912 2011-09-13 05:34:52,434 WARN org.apache.hadoop.hbase.regionserver.wal.HLog: IPC Server handler 28 on 60020 took 1052 ms appending an edit to hlog; editcount=2978, len~=12.7k 2011-09-13 05:34:55,782 WARN org.apache.hadoop.hdfs.DFSClient: Error while syncing java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1041) 2011-09-13 05:34:55,782 FATAL org.apache.hadoop.hbase.regionserver.wal.HLog: Could not sync. Requesting close of hlog java.io.IOException: Reflection at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:194) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1041) Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) ... 3 more Caused by: java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) ... 7 more 2011-09-13 05:34:55,783 ERROR org.apache.hadoop.hbase.regionserver.wal.HLog: Error while syncing, requesting close of hlog java.io.IOException: Reflection at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:194) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1165) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1041) Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:192) ... 3 more Caused by: java.io.IOException: DFSOutputStream is closed at org.apache.hadoop.hdfs.DFSOutputStream.isClosed(DFSOutputStream.java:1186) at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:1590) at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1478) at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:123) ... 7 more 2011-09-13 05:34:55,784 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315917280910, entries=3819, filesize=64560773. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315917295370 2011-09-13 05:35:08,040 WARN org.apache.hadoop.hbase.regionserver.wal.HLog: IPC Server handler 68 on 60020 took 1209 ms appending an edit to hlog; editcount=3469, len~=18.4k 2011-09-13 05:35:09,817 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315917295370, entries=3777, filesize=64398471. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315917308847 2011-09-13 05:35:09,904 WARN org.apache.hadoop.hbase.regionserver.MemStoreFlusher: Region usertable,user1022866120,1315898727279.f3cb8bb14102b43abbb2b45cd2fc4945. has too many store files; delaying flush up to 90000ms 2011-09-13 05:35:16,661 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315917308847, entries=3788, filesize=64484712. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315917316060 2011-09-13 05:35:23,457 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 5 file(s) in family of usertable,user224923440,1315898727290.766e5174f1140d766dd0af91cbd1d845.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/766e5174f1140d766dd0af91cbd1d845/family/7492436267646903170, size=391.6m; total size for store is 2.4g 2011-09-13 05:35:23,457 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user224923440,1315898727290.766e5174f1140d766dd0af91cbd1d845., storeName=family, fileCount=5, fileSize=391.6m, priority=-3, time=3931539535599297; duration=1mins, 18sec 2011-09-13 05:35:23,458 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user2029666280,1315898727289.8cb227d0fbbb2ad66e2f3d25cdfa0f89. 2011-09-13 05:35:23,458 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 4 file(s) in family of usertable,user2029666280,1315898727289.8cb227d0fbbb2ad66e2f3d25cdfa0f89. into hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/.tmp, seqid=2212169, totalSize=370.4m 2011-09-13 05:35:31,581 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/d057127fe98e78ea72dc9be657e68a4a/.tmp/4219626671725579973 to hdfs://c0309.xxxxx:17020/hbase/usertable/d057127fe98e78ea72dc9be657e68a4a/family/2141222962234534313 2011-09-13 05:35:31,626 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/d057127fe98e78ea72dc9be657e68a4a/family/2141222962234534313, entries=204770, sequenceid=2262390, memsize=64.0m, filesize=28.5m 2011-09-13 05:35:31,858 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.0m for region usertable,user1397738520,1315898727283.d057127fe98e78ea72dc9be657e68a4a. in 5159ms, sequenceid=2262390, compaction requested=true 2011-09-13 05:35:35,962 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/8c36971138a4bfebe5aeb94c163f2a81/.tmp/6179773773046961234 to hdfs://c0309.xxxxx:17020/hbase/usertable/8c36971138a4bfebe5aeb94c163f2a81/family/5489477654829179203 2011-09-13 05:35:35,999 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/8c36971138a4bfebe5aeb94c163f2a81/family/5489477654829179203, entries=204930, sequenceid=2262967, memsize=64.1m, filesize=28.5m 2011-09-13 05:35:36,004 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.1m for region usertable,user1274566160,1315898727282.8c36971138a4bfebe5aeb94c163f2a81. in 3921ms, sequenceid=2262967, compaction requested=true 2011-09-13 05:35:36,051 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315917316060, entries=4754, filesize=80754364. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315917336004 2011-09-13 05:35:50,147 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/169b91a09b9b10a23602aaea5e8b9fc2/.tmp/824625477084292547 to hdfs://c0309.xxxxx:17020/hbase/usertable/169b91a09b9b10a23602aaea5e8b9fc2/family/357322295673965458 2011-09-13 05:35:50,216 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/169b91a09b9b10a23602aaea5e8b9fc2/family/357322295673965458, entries=205840, sequenceid=2265228, memsize=64.4m, filesize=28.7m 2011-09-13 05:35:50,335 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.4m for region usertable,user1901138600,1315898727288.169b91a09b9b10a23602aaea5e8b9fc2. in 5595ms, sequenceid=2265228, compaction requested=false 2011-09-13 05:35:50,335 WARN org.apache.hadoop.hbase.regionserver.MemStoreFlusher: Region usertable,user1146038480,1315898727281.f6d7a2cd3251b638e0f60e6e46e71a88. has too many store files; delaying flush up to 90000ms 2011-09-13 05:36:03,487 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/50d20bc0e631f85cd8b961fcf983914a/.tmp/7539279640948428298 to hdfs://c0309.xxxxx:17020/hbase/usertable/50d20bc0e631f85cd8b961fcf983914a/family/2602617292937918800 2011-09-13 05:36:03,544 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/50d20bc0e631f85cd8b961fcf983914a/family/2602617292937918800, entries=211750, sequenceid=2266460, memsize=66.2m, filesize=29.5m 2011-09-13 05:36:03,548 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~66.2m for region usertable,user1526266200,1315898727285.50d20bc0e631f85cd8b961fcf983914a. in 13213ms, sequenceid=2266460, compaction requested=true 2011-09-13 05:36:03,548 WARN org.apache.hadoop.hbase.regionserver.MemStoreFlusher: Region usertable,user1649438560,1315898727286.29ec66424227dc3cbe3d8b2311764fc9. has too many store files; delaying flush up to 90000ms 2011-09-13 05:36:03,548 WARN org.apache.hadoop.hbase.regionserver.MemStoreFlusher: Region usertable,user1772610920,1315898727287.77b5b2f51884554ebf951146d25f1bd2. has too many store files; delaying flush up to 90000ms 2011-09-13 05:36:03,548 WARN org.apache.hadoop.hbase.regionserver.MemStoreFlusher: Region usertable,user2029666280,1315898727289.8cb227d0fbbb2ad66e2f3d25cdfa0f89. has too many store files; delaying flush up to 90000ms 2011-09-13 05:36:10,037 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/.tmp/1116291469831420235 to hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/family/6594578935249814369 2011-09-13 05:36:10,284 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/521c139985d9918a22c08430ef00e51d/family/6594578935249814369, entries=214510, sequenceid=2266829, memsize=66.9m, filesize=29.7m 2011-09-13 05:36:11,463 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~66.9m for region usertable,user353451120,1315898727292.521c139985d9918a22c08430ef00e51d. in 7915ms, sequenceid=2266829, compaction requested=true 2011-09-13 05:36:17,424 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/766e5174f1140d766dd0af91cbd1d845/.tmp/7799920057488547187 to hdfs://c0309.xxxxx:17020/hbase/usertable/766e5174f1140d766dd0af91cbd1d845/family/1992173516183315686 2011-09-13 05:36:17,476 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/766e5174f1140d766dd0af91cbd1d845/family/1992173516183315686, entries=213290, sequenceid=2267548, memsize=66.6m, filesize=29.5m 2011-09-13 05:36:17,534 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~66.6m for region usertable,user224923440,1315898727290.766e5174f1140d766dd0af91cbd1d845. in 6070ms, sequenceid=2267548, compaction requested=false 2011-09-13 05:36:17,534 WARN org.apache.hadoop.hbase.regionserver.MemStoreFlusher: Region usertable,user980023560,1315898727298.a4521b468fb6ac94dd495ee2db515ae0. has too many store files; delaying flush up to 90000ms 2011-09-13 05:36:17,534 WARN org.apache.hadoop.hbase.regionserver.MemStoreFlusher: Region usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd. has too many store files; delaying flush up to 90000ms 2011-09-13 05:36:17,709 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: Roll /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315917336004, entries=5034, filesize=85305710. New hlog /hbase/.logs/c0334.xxxxx,60020,1315897876046/c0334.xxxxx%2C60020%2C1315897876046.1315917377534 2011-09-13 05:36:24,944 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/.tmp/2884860961940745686 to hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/family/1028855939866383107 2011-09-13 05:36:24,985 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/family/1028855939866383107, entries=205880, sequenceid=2269838, memsize=64.2m, filesize=28.5m 2011-09-13 05:36:26,044 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~64.2m for region usertable,user728323520,1315898727295.a623bd664ea942437bf6ae1073fb5560. in 3213ms, sequenceid=2269838, compaction requested=true 2011-09-13 05:36:26,045 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user728323520,1315898727295.a623bd664ea942437bf6ae1073fb5560. 2011-09-13 05:36:26,045 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 3 file(s) in family of usertable,user728323520,1315898727295.a623bd664ea942437bf6ae1073fb5560. into hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/.tmp, seqid=2269838, totalSize=85.5m 2011-09-13 05:36:32,692 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 3 file(s) in family of usertable,user728323520,1315898727295.a623bd664ea942437bf6ae1073fb5560.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/a623bd664ea942437bf6ae1073fb5560/family/6153167156662770205, size=85.5m; total size for store is 2.5g 2011-09-13 05:36:32,692 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user728323520,1315898727295.a623bd664ea942437bf6ae1073fb5560., storeName=family, fileCount=3, fileSize=85.5m, priority=0, time=3931687301808297; duration=6sec 2011-09-13 05:36:40,484 INFO org.apache.hadoop.hbase.regionserver.MemStoreFlusher: Waited 90580ms on a compaction to clean up 'too many store files'; waited long enough... proceeding with flush of usertable,user1022866120,1315898727279.f3cb8bb14102b43abbb2b45cd2fc4945. 2011-09-13 05:36:50,152 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/.tmp/3648040112180430939 to hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/family/1015441141255914512 2011-09-13 05:36:50,208 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/family/1015441141255914512, entries=320100, sequenceid=2270688, memsize=100.1m, filesize=44.6m 2011-09-13 05:36:50,212 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~100.1m for region usertable,user1022866120,1315898727279.f3cb8bb14102b43abbb2b45cd2fc4945. in 9727ms, sequenceid=2270688, compaction requested=true 2011-09-13 05:36:50,212 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user1022866120,1315898727279.f3cb8bb14102b43abbb2b45cd2fc4945. 2011-09-13 05:36:50,213 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 3 file(s) in family of usertable,user1022866120,1315898727279.f3cb8bb14102b43abbb2b45cd2fc4945. into hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/.tmp, seqid=2270688, totalSize=101.9m 2011-09-13 05:37:06,043 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 3 file(s) in family of usertable,user1022866120,1315898727279.f3cb8bb14102b43abbb2b45cd2fc4945.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/f3cb8bb14102b43abbb2b45cd2fc4945/family/4549606098359970961, size=101.9m; total size for store is 2.5g 2011-09-13 05:37:06,043 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user1022866120,1315898727279.f3cb8bb14102b43abbb2b45cd2fc4945., storeName=family, fileCount=3, fileSize=101.9m, priority=-2, time=3931711469770297; duration=15sec 2011-09-13 05:37:14,943 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 4 file(s) in family of usertable,user2029666280,1315898727289.8cb227d0fbbb2ad66e2f3d25cdfa0f89.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/family/1773951031429929627, size=370.4m; total size for store is 2.5g 2011-09-13 05:37:14,943 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user2029666280,1315898727289.8cb227d0fbbb2ad66e2f3d25cdfa0f89., storeName=family, fileCount=4, fileSize=370.4m, priority=-2, time=3931319082949297; duration=1mins, 51sec 2011-09-13 05:37:14,943 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user1146038480,1315898727281.f6d7a2cd3251b638e0f60e6e46e71a88. 2011-09-13 05:37:14,944 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 4 file(s) in family of usertable,user1146038480,1315898727281.f6d7a2cd3251b638e0f60e6e46e71a88. into hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/.tmp, seqid=2240402, totalSize=360.9m 2011-09-13 05:37:17,477 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/.tmp/7460020043193899798 to hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/family/1985542567494013775 2011-09-13 05:37:17,594 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/8cb227d0fbbb2ad66e2f3d25cdfa0f89/family/1985542567494013775, entries=245490, sequenceid=2270899, memsize=76.8m, filesize=34.2m 2011-09-13 05:37:17,596 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~76.8m for region usertable,user2029666280,1315898727289.8cb227d0fbbb2ad66e2f3d25cdfa0f89. in 2124ms, sequenceid=2270899, compaction requested=true 2011-09-13 05:37:18,499 INFO org.apache.hadoop.hbase.regionserver.MemStoreFlusher: Waited 90363ms on a compaction to clean up 'too many store files'; waited long enough... proceeding with flush of usertable,user1146038480,1315898727281.f6d7a2cd3251b638e0f60e6e46e71a88. 2011-09-13 05:37:19,213 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/.tmp/6285726251815186846 to hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/family/9088947628750133852 2011-09-13 05:37:19,267 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/family/9088947628750133852, entries=245940, sequenceid=2270900, memsize=76.9m, filesize=34.3m 2011-09-13 05:37:19,286 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~76.9m for region usertable,user1146038480,1315898727281.f6d7a2cd3251b638e0f60e6e46e71a88. in 787ms, sequenceid=2270900, compaction requested=false 2011-09-13 05:37:20,188 INFO org.apache.hadoop.hbase.regionserver.MemStoreFlusher: Waited 90070ms on a compaction to clean up 'too many store files'; waited long enough... proceeding with flush of usertable,user1772610920,1315898727287.77b5b2f51884554ebf951146d25f1bd2. 2011-09-13 05:37:26,998 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/.tmp/103695522217572670 to hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/family/5602668495465755287 2011-09-13 05:37:27,042 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/77b5b2f51884554ebf951146d25f1bd2/family/5602668495465755287, entries=244670, sequenceid=2270901, memsize=76.5m, filesize=34.1m 2011-09-13 05:37:27,046 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~76.5m for region usertable,user1772610920,1315898727287.77b5b2f51884554ebf951146d25f1bd2. in 6857ms, sequenceid=2270901, compaction requested=true 2011-09-13 05:37:27,046 INFO org.apache.hadoop.hbase.regionserver.MemStoreFlusher: Waited 94985ms on a compaction to clean up 'too many store files'; waited long enough... proceeding with flush of usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd. 2011-09-13 05:37:34,723 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/.tmp/6523962375261811866 to hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/family/4091419921002399825 2011-09-13 05:37:34,782 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/family/4091419921002399825, entries=238410, sequenceid=2270902, memsize=74.4m, filesize=33.0m 2011-09-13 05:37:34,788 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~74.4m for region usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd. in 7742ms, sequenceid=2270902, compaction requested=true 2011-09-13 05:37:34,788 INFO org.apache.hadoop.hbase.regionserver.MemStoreFlusher: Waited 103819ms on a compaction to clean up 'too many store files'; waited long enough... proceeding with flush of usertable,user980023560,1315898727298.a4521b468fb6ac94dd495ee2db515ae0. 2011-09-13 05:37:34,788 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on family in region usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd. 2011-09-13 05:37:34,788 INFO org.apache.hadoop.hbase.regionserver.Store: Starting compaction of 3 file(s) in family of usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd. into hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/.tmp, seqid=2270902, totalSize=92.0m 2011-09-13 05:37:36,881 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/.tmp/1963282754545688019 to hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/family/501489172408919831 2011-09-13 05:37:36,943 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/a4521b468fb6ac94dd495ee2db515ae0/family/501489172408919831, entries=245240, sequenceid=2270903, memsize=76.5m, filesize=33.9m 2011-09-13 05:37:36,948 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~76.5m for region usertable,user980023560,1315898727298.a4521b468fb6ac94dd495ee2db515ae0. in 2160ms, sequenceid=2270903, compaction requested=false 2011-09-13 05:37:36,948 INFO org.apache.hadoop.hbase.regionserver.MemStoreFlusher: Waited 106938ms on a compaction to clean up 'too many store files'; waited long enough... proceeding with flush of usertable,user1649438560,1315898727286.29ec66424227dc3cbe3d8b2311764fc9. 2011-09-13 05:37:39,195 INFO org.apache.hadoop.hbase.regionserver.Store: Renaming flushed file at hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/.tmp/922406032943891596 to hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/family/7595271852690338206 2011-09-13 05:37:39,257 INFO org.apache.hadoop.hbase.regionserver.Store: Added hdfs://c0309.xxxxx:17020/hbase/usertable/29ec66424227dc3cbe3d8b2311764fc9/family/7595271852690338206, entries=242980, sequenceid=2270904, memsize=76.0m, filesize=33.8m 2011-09-13 05:37:39,264 INFO org.apache.hadoop.hbase.regionserver.HRegion: Finished memstore flush of ~76.0m for region usertable,user1649438560,1315898727286.29ec66424227dc3cbe3d8b2311764fc9. in 2316ms, sequenceid=2270904, compaction requested=true 2011-09-13 05:37:44,918 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 3 file(s) in family of usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/5efab8f898cb78044c8833198bfd48dd/family/8885489610922137693, size=92.0m; total size for store is 2.5g 2011-09-13 05:37:44,918 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user599795840,1315898727294.5efab8f898cb78044c8833198bfd48dd., storeName=family, fileCount=3, fileSize=92.0m, priority=-2, time=3931756045636297; duration=10sec 2011-09-13 05:37:49,993 INFO org.apache.hadoop.hbase.regionserver.Store: Completed compaction of 4 file(s) in family of usertable,user1146038480,1315898727281.f6d7a2cd3251b638e0f60e6e46e71a88.; new storefile name=hdfs://c0309.xxxxx:17020/hbase/usertable/f6d7a2cd3251b638e0f60e6e46e71a88/family/4372165091520159779, size=360.9m; total size for store is 2.5g 2011-09-13 05:37:49,993 INFO org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest: completed compaction: regionName=usertable,user1146038480,1315898727281.f6d7a2cd3251b638e0f60e6e46e71a88., storeName=family, fileCount=4, fileSize=360.9m, priority=-2, time=3931484873019297; duration=35sec