2016-04-13 07:41:11,263 INFO [10.0.0.75,16000,1460531913905_ChoreService_1] web.PrivateAzureDataLakeFileSystem: { "Type" : "Network","TotalTimeSpendInMs" : 94,"ResponseCode" : 200,"Response" : "OK","ServerPerf" : "[8bfd234e-0de9-48fa-b0fe-8ae0a085549b][ AuthTime::1488.37654900223::PostAuthTime::30960.7983857102 ][GetFileStatus :: 00:00:000 ms]%0a[HdfsGetFileStatusV2 :: 00:00:052 ms]%0a[GetFileStatus :: 00:00:053 ms]%0a[GETFILESTATUS :: 00:00:053 ms]%0a","Machine" : "bn4sch103190624","RequestStartTime" : "13/04/16 07:41:11.169","SSLHandshakeCompleteTime" : "13/04/16 07:41:11.170","SSLHandshakeCompleteTimeInMs" : 1,"RequestEndTime" : "13/04/16 07:41:11.263","URL" : "https://xy4.a.net:443/webhdfs/v1/hbase/.hbase-snapshot/.tmp/snapshot_tb1_create/data.manifest?op=GETFILESTATUS&user.name=hbase","WebhdfsObjectId" : "a36d695c-52cb-452d-b9ca-853393978413","OAuthTokenRequest" : "13/04/16 07:41:10.979","OAuthTokenResponse" : "13/04/16 07:41:10.979","OAuthRefreshTokenTimeInMs" : 0,"Host" : "zk2-hbase4","LogVersion" : "1.2.2-Dogfood"} 2016-04-13 07:41:11,264 INFO [10.0.0.75,16000,1460531913905_ChoreService_1] web.PrivateAzureDataLakeFileSystem: { "Type" : "API","TotalTimeSpendInMs" : 298,"Method" : "getFileStatus","jobid" : "NA","WebhdfsObjectId" : "0d02ed5d-c694-497f-9000-dad71994b695","Start" : "13/04/16 07:41:10.966","End" : "13/04/16 07:41:11.264","PathInfo" : "adl://xy4.a.net:443/hbase/.hbase-snapshot/.tmp/snapshot_tb1_create/data.manifest","Result" : "FileStatus{path=adl://xy4.a.net:443/hbase/.hbase-snapshot/.tmp/snapshot_tb1_create/data.manifest; isDirectory=false; length=0; replication=1; blocksize=268435456; modification_time=1460533271137; access_time=1460533270843; owner=hbase; group=hdfs; permission=rwxrwx---; isSymlink=false}","LogVersion" : "1.2.2-Dogfood"} 2016-04-13 07:41:11,264 INFO [10.0.0.75,16000,1460531913905_ChoreService_1] web.PrivateAzureDataLakeFileSystem: BatchByteArrayInputStream Construction Time 298 2016-04-13 07:41:11,264 INFO [10.0.0.75,16000,1460531913905_ChoreService_1] web.PrivateAzureDataLakeFileSystem: FSDataInputStream Construction Time 0 2016-04-13 07:41:11,264 INFO [10.0.0.75,16000,1460531913905_ChoreService_1] web.PrivateAzureDataLakeFileSystem: { "Type" : "API","TotalTimeSpendInMs" : 298,"Method" : "OPEN","jobid" : "NA","WebhdfsObjectId" : "0d02ed5d-c694-497f-9000-dad71994b695","Start" : "13/04/16 07:41:10.966","End" : "13/04/16 07:41:11.264","PathInfo" : "adl://xy4.a.net:443/hbase/.hbase-snapshot/.tmp/snapshot_tb1_create/data.manifest","Result" : "NA","LogVersion" : "1.2.2-Dogfood"} 2016-04-13 07:41:11,265 INFO [10.0.0.75,16000,1460531913905_ChoreService_1] web.PrivateAzureDataLakeFileSystem: Initial Fill 2016-04-13 07:41:11,274 ERROR [10.0.0.75,16000,1460531913905_ChoreService_1] cleaner.HFileCleaner: Caught error java.lang.ExceptionInInitializerError at org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos$SnapshotDataManifest.internalGetFieldAccessorTable(SnapshotProtos.java:3883) at com.google.protobuf.GeneratedMessage.getDescriptorForType(GeneratedMessage.java:98) at com.google.protobuf.AbstractMessage$Builder.findMissingFields(AbstractMessage.java:789) at com.google.protobuf.AbstractMessage$Builder.findMissingFields(AbstractMessage.java:780) at com.google.protobuf.AbstractMessage$Builder.newUninitializedMessageException(AbstractMessage.java:770) at com.google.protobuf.AbstractMessage.newUninitializedMessageException(AbstractMessage.java:237) at com.google.protobuf.AbstractParser.newUninitializedMessageException(AbstractParser.java:57) at com.google.protobuf.AbstractParser.checkMessageInitialized(AbstractParser.java:71) at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:217) at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:223) at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:49) at org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos$SnapshotDataManifest.parseFrom(SnapshotProtos.java:4094) at org.apache.hadoop.hbase.snapshot.SnapshotManifest.readDataManifest(SnapshotManifest.java:433) at org.apache.hadoop.hbase.snapshot.SnapshotManifest.load(SnapshotManifest.java:273) at org.apache.hadoop.hbase.snapshot.SnapshotManifest.open(SnapshotManifest.java:119) at org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil.visitTableStoreFiles(SnapshotReferenceUtil.java:125) at org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil.getHFileNames(SnapshotReferenceUtil.java:346) at org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil.getHFileNames(SnapshotReferenceUtil.java:329) at org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner$1.filesUnderSnapshot(SnapshotHFileCleaner.java:85) at org.apache.hadoop.hbase.master.snapshot.SnapshotFileCache.getSnapshotsInProgress(SnapshotFileCache.java:303) at org.apache.hadoop.hbase.master.snapshot.SnapshotFileCache.getUnreferencedFiles(SnapshotFileCache.java:194) at org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner.getDeletableFiles(SnapshotHFileCleaner.java:62) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.checkAndDeleteFiles(CleanerChore.java:233) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.checkAndDeleteEntries(CleanerChore.java:157) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.checkAndDeleteDirectory(CleanerChore.java:180) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.checkAndDeleteEntries(CleanerChore.java:149) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.checkAndDeleteDirectory(CleanerChore.java:180) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.checkAndDeleteEntries(CleanerChore.java:149) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.checkAndDeleteDirectory(CleanerChore.java:180) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.checkAndDeleteEntries(CleanerChore.java:149) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.checkAndDeleteDirectory(CleanerChore.java:180) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.checkAndDeleteEntries(CleanerChore.java:149) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.checkAndDeleteDirectory(CleanerChore.java:180) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.checkAndDeleteEntries(CleanerChore.java:149) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.chore(CleanerChore.java:124) at org.apache.hadoop.hbase.ScheduledChore.run(ScheduledChore.java:185) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) Caused by: java.lang.IllegalArgumentException: Invalid embedded descriptor for "Snapshot.proto". at com.google.protobuf.Descriptors$FileDescriptor.internalBuildGeneratedFileFrom(Descriptors.java:301) at org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.(SnapshotProtos.java:4780) ... 43 more Caused by: com.google.protobuf.Descriptors$DescriptorValidationException: hbase.pb.SnapshotRegionManifest.StoreFile.reference: ".hbase.pb.Reference" is not defined. at com.google.protobuf.Descriptors$DescriptorPool.lookupSymbol(Descriptors.java:1784) at com.google.protobuf.Descriptors$FieldDescriptor.crossLink(Descriptors.java:982) at com.google.protobuf.Descriptors$FieldDescriptor.access$700(Descriptors.java:658) at com.google.protobuf.Descriptors$Descriptor.crossLink(Descriptors.java:625) at com.google.protobuf.Descriptors$Descriptor.crossLink(Descriptors.java:621) at com.google.protobuf.Descriptors$Descriptor.access$500(Descriptors.java:440) at com.google.protobuf.Descriptors$FileDescriptor.crossLink(Descriptors.java:394) at com.google.protobuf.Descriptors$FileDescriptor.buildFrom(Descriptors.java:252) at com.google.protobuf.Descriptors$FileDescriptor.internalBuildGeneratedFileFrom(Descriptors.java:299) ... 44 more 2016-04-13 07:41:11,276 INFO [10.0.0.75,16000,1460531913905_ChoreService_1] cleaner.HFileCleaner: Chore: HFileCleaner missed its start time 2016-04-13 07:41:11,299 INFO [MASTER_TABLE_OPERATIONS-10.0.0.75:16000-0] web.PrivateAzureDataLakeFileSystem: { "Type" : "Network","TotalTimeSpendInMs" : 198,"ResponseCode" : 200,"Response" : "OK","ServerPerf" : "[58c0f8fc-42be-4f46-90c5-e25ef2192709][ AuthTime::626.14515276402::PostAuthTime::133.868465037663 ][GetCacheEntry :: 00:00:000 ms]%0a[FsOpenStream :: 00:00:027 ms]%0a[OpenNewHandle :: 00:00:028 ms]%0a[CreateCacheEntry :: 00:00:000 ms]%0a[GetOrCreateHandle :: 00:00:028 ms]%0a[Open :: 00:00:028 ms]%0a[FsAppendStream :: 00:00:158 ms]%0a[Write :: 00:00:159 ms]%0a[APPEND :: 00:00:188 ms]%0a","Machine" : "bn4sch103200823","RequestStartTime" : "13/04/16 07:41:11.101","SSLHandshakeCompleteTime" : "13/04/16 07:41:11.103","SSLHandshakeCompleteTimeInMs" : 2,"RequestEndTime" : "13/04/16 07:41:11.299","URL" : "https://xy4.a.net:443/webhdfs/v1/hbase/.hbase-snapshot/.tmp/snapshot_tb1_create/data.manifest?op=APPEND&user.name=hbase&ADLFeatureSet=1.2.2-Dogfood&append=true&blocksize=134217728&buffersize=131072&fileSessionId=32f912a3-8f3f-46aa-8355-aeba28bbe03e&flush=true&offset=0&overwrite=true&permission=644&replication=3","WebhdfsObjectId" : "a36d695c-52cb-452d-b9ca-853393978413","OAuthTokenRequest" : "13/04/16 07:41:10.898","OAuthTokenResponse" : "13/04/16 07:41:10.898","OAuthRefreshTokenTimeInMs" : 0,"Host" : "zk2-hbase4","LogVersion" : "1.2.2-Dogfood"} 2016-04-13 07:41:11,299 INFO [MASTER_TABLE_OPERATIONS-10.0.0.75:16000-0] web.PrivateAzureDataLakeFileSystem: Total bytes Written : 953 [adl://xy4.a.net:443/hbase/.hbase-snapshot/.tmp/snapshot_tb1_create/data.manifest] 2016-04-13 07:41:11,310 INFO [pool-4326-thread-1] adal4j.UserDiscoveryRequest: [Correlation ID: null] Sent (null) Correlation Id is not same as received (48eddb18-8d2a-4dcb-8227-39b60de0a354). 2016-04-13 07:41:11,311 INFO [pool-4326-thread-1] adal4j.AuthenticationAuthority: [Correlation ID: c214c39a-61e9-4e9b-a7db-bdc839b8901c] Instance discovery was successful 2016-04-13 07:41:11,336 INFO [pool-4327-thread-1] adal4j.UserDiscoveryRequest: [Correlation ID: null] Sent (null) Correlation Id is not same as received (c3717c20-b5d5-4dc1-80fd-353c8f0f8982). 2016-04-13 07:41:11,336 INFO [pool-4327-thread-1] adal4j.AuthenticationAuthority: [Correlation ID: b8348cd0-f136-4631-b677-15a3d0f8a5ee] Instance discovery was successful 2016-04-13 07:41:11,599 INFO [10.0.0.75,16000,1460531913905_ChoreService_2] web.PrivateAzureDataLakeFileSystem: { "Type" : "Network","TotalTimeSpendInMs" : 112,"ResponseCode" : 200,"Response" : "OK","ServerPerf" : "[8f954048-23ea-4899-96c5-9a7f6d4759e6][ AuthTime::1385.30338101275::PostAuthTime::30415.5140292441 ][HdfsListStatus :: 00:00:073 ms]%0a[ListStatus :: 00:00:073 ms]%0a[LISTSTATUS :: 00:00:073 ms]%0a","Machine" : "bn4sch103200823","RequestStartTime" : "13/04/16 07:41:11.487","SSLHandshakeCompleteTime" : "13/04/16 07:41:11.487","SSLHandshakeCompleteTimeInMs" : 0,"RequestEndTime" : "13/04/16 07:41:11.599","URL" : "https://xy4.a.net:443/webhdfs/v1/hbase/archive?op=LISTSTATUS&user.name=hbase","WebhdfsObjectId" : "a36d695c-52cb-452d-b9ca-853393978413","OAuthTokenRequest" : "13/04/16 07:41:11.275","OAuthTokenResponse" : "13/04/16 07:41:11.276","OAuthRefreshTokenTimeInMs" : 1,"Host" : "zk2-hbase4","LogVersion" : "1.2.2-Dogfood"} 2016-04-13 07:41:11,600 INFO [10.0.0.75,16000,1460531913905_ChoreService_2] web.PrivateAzureDataLakeFileSystem: { "Type" : "API","TotalTimeSpendInMs" : 325,"Method" : "listStatus","jobid" : "NA","WebhdfsObjectId" : "0d02ed5d-c694-497f-9000-dad71994b695","Start" : "13/04/16 07:41:11.275","End" : "13/04/16 07:41:11.600","PathInfo" : "adl://xy4.a.net:443/hbase/archive","Result" : "[Lorg.apache.hadoop.fs.FileStatus;@903aeff","LogVersion" : "1.2.2-Dogfood"} 2016-04-13 07:41:11,635 INFO [pool-4328-thread-1] adal4j.UserDiscoveryRequest: [Correlation ID: null] Sent (null) Correlation Id is not same as received (0c001cc5-f2eb-4dfb-80f5-e3dcde7e75d7). 2016-04-13 07:41:11,636 INFO [pool-4328-thread-1] adal4j.AuthenticationAuthority: [Correlation ID: b4d31beb-c7f0-42e3-8248-aa5c9a9c55df] Instance discovery was successful 2016-04-13 07:41:11,790 INFO [MASTER_TABLE_OPERATIONS-10.0.0.75:16000-0] web.PrivateAzureDataLakeFileSystem: { "Type" : "Network","TotalTimeSpendInMs" : 276,"ResponseCode" : 200,"Response" : "OK","ServerPerf" : "[30fccf9b-bd70-4214-a5e7-d429dd51813b][ AuthTime::562.418149982165::PostAuthTime::118.471351745293 ][FsDelete :: 00:00:268 ms]%0a[CleanCacheDelete :: 00:00:000 ms]%0a[Delete : Recursive :: 00:00:268 ms]%0a[DELETE :: 00:00:268 ms]%0a","Machine" : "bn4sch103190624","RequestStartTime" : "13/04/16 07:41:11.514","SSLHandshakeCompleteTime" : "13/04/16 07:41:11.514","SSLHandshakeCompleteTimeInMs" : 0,"RequestEndTime" : "13/04/16 07:41:11.790","URL" : "https://xy4.a.net:443/webhdfs/v1/hbase/.hbase-snapshot/.tmp/snapshot_tb1_create/region-manifest.112a30b8c6c8aa67858d22419409af37?op=DELETE&user.name=hbase&recursive=true","WebhdfsObjectId" : "a36d695c-52cb-452d-b9ca-853393978413","OAuthTokenRequest" : "13/04/16 07:41:11.300","OAuthTokenResponse" : "13/04/16 07:41:11.300","OAuthRefreshTokenTimeInMs" : 0,"Host" : "zk2-hbase4","LogVersion" : "1.2.2-Dogfood"} 2016-04-13 07:41:11,790 INFO [MASTER_TABLE_OPERATIONS-10.0.0.75:16000-0] web.PrivateAzureDataLakeFileSystem: { "Type" : "API","TotalTimeSpendInMs" : 491,"Method" : "delete","jobid" : "NA","WebhdfsObjectId" : "0d02ed5d-c694-497f-9000-dad71994b695","Start" : "13/04/16 07:41:11.299","End" : "13/04/16 07:41:11.790","PathInfo" : "adl://xy4.a.net:443/hbase/.hbase-snapshot/.tmp/snapshot_tb1_create/region-manifest.112a30b8c6c8aa67858d22419409af37","Result" : "true","LogVersion" : "1.2.2-Dogfood"} 2016-04-13 07:41:11,791 INFO [MASTER_TABLE_OPERATIONS-10.0.0.75:16000-0] web.PrivateAzureDataLakeFileSystem: statistics report Time 0 2016-04-13 07:41:11,791 INFO [MASTER_TABLE_OPERATIONS-10.0.0.75:16000-0] web.PrivateAzureDataLakeFileSystem: URL Construction Time 0 2016-04-13 07:41:11,792 INFO [MASTER_TABLE_OPERATIONS-10.0.0.75:16000-0] web.PrivateAzureDataLakeFileSystem: Cached Instance Found : adl://xy4.a.net:443/hbase/.hbase-snapshot/.tmp/snapshot_tb1_create/data.manifest 2016-04-13 07:41:11,792 INFO [MASTER_TABLE_OPERATIONS-10.0.0.75:16000-0] web.PrivateAzureDataLakeFileSystem: { "Type" : "API","TotalTimeSpendInMs" : 0,"Method" : "getFileStatus","jobid" : "NA","WebhdfsObjectId" : "0d02ed5d-c694-497f-9000-dad71994b695","Start" : "13/04/16 07:41:11.792","End" : "13/04/16 07:41:11.792","PathInfo" : "adl://xy4.a.net:443/hbase/.hbase-snapshot/.tmp/snapshot_tb1_create/data.manifest","Result" : "FileStatus{path=adl://xy4.a.net:443/hbase/.hbase-snapshot/.tmp/snapshot_tb1_create/data.manifest; isDirectory=false; length=0; replication=1; blocksize=268435456; modification_time=1460533271137; access_time=1460533270843; owner=hbase; group=hdfs; permission=rwxrwx---; isSymlink=false}","LogVersion" : "1.2.2-Dogfood"} 2016-04-13 07:41:11,793 INFO [MASTER_TABLE_OPERATIONS-10.0.0.75:16000-0] web.PrivateAzureDataLakeFileSystem: BatchByteArrayInputStream Construction Time 1 2016-04-13 07:41:11,793 INFO [MASTER_TABLE_OPERATIONS-10.0.0.75:16000-0] web.PrivateAzureDataLakeFileSystem: FSDataInputStream Construction Time 0 2016-04-13 07:41:11,794 INFO [MASTER_TABLE_OPERATIONS-10.0.0.75:16000-0] web.PrivateAzureDataLakeFileSystem: { "Type" : "API","TotalTimeSpendInMs" : 3,"Method" : "OPEN","jobid" : "NA","WebhdfsObjectId" : "0d02ed5d-c694-497f-9000-dad71994b695","Start" : "13/04/16 07:41:11.791","End" : "13/04/16 07:41:11.794","PathInfo" : "adl://xy4.a.net:443/hbase/.hbase-snapshot/.tmp/snapshot_tb1_create/data.manifest","Result" : "NA","LogVersion" : "1.2.2-Dogfood"} 2016-04-13 07:41:11,794 INFO [MASTER_TABLE_OPERATIONS-10.0.0.75:16000-0] web.PrivateAzureDataLakeFileSystem: Initial Fill 2016-04-13 07:41:11,796 DEBUG [MASTER_TABLE_OPERATIONS-10.0.0.75:16000-0] snapshot.TakeSnapshotHandler: Launching cleanup of working dir:adl://xy4.a.net:443/hbase/.hbase-snapshot/.tmp/snapshot_tb1_create 2016-04-13 07:41:11,796 INFO [MASTER_TABLE_OPERATIONS-10.0.0.75:16000-0] web.PrivateAzureDataLakeFileSystem: Cached Instance Found : adl://xy4.a.net:443/hbase/.hbase-snapshot/.tmp/snapshot_tb1_create 2016-04-13 07:41:11,803 INFO [MASTER_TABLE_OPERATIONS-10.0.0.75:16000-0] web.PrivateAzureDataLakeFileSystem: { "Type" : "API","TotalTimeSpendInMs" : 7,"Method" : "getFileStatus","jobid" : "NA","WebhdfsObjectId" : "0d02ed5d-c694-497f-9000-dad71994b695","Start" : "13/04/16 07:41:11.796","End" : "13/04/16 07:41:11.803","PathInfo" : "adl://xy4.a.net:443/hbase/.hbase-snapshot/.tmp/snapshot_tb1_create","Result" : "FileStatus{path=adl://xy4.a.net:443/hbase/.hbase-snapshot/.tmp/snapshot_tb1_create; isDirectory=true; modification_time=1460533269080; access_time=1460533254767; owner=hbase; group=hdfs; permission=rwxrwx---; isSymlink=false}","LogVersion" : "1.2.2-Dogfood"} 2016-04-13 07:41:11,842 INFO [pool-4329-thread-1] adal4j.UserDiscoveryRequest: [Correlation ID: null] Sent (null) Correlation Id is not same as received (534edf2b-40cc-4f89-981f-ad23cf38d1bc). 2016-04-13 07:41:11,842 INFO [pool-4329-thread-1] adal4j.AuthenticationAuthority: [Correlation ID: 217cdcfd-dca6-47fb-9961-9b33b412b4cc] Instance discovery was successful 2016-04-13 07:41:11,914 INFO [10.0.0.75,16000,1460531913905_ChoreService_2] web.PrivateAzureDataLakeFileSystem: { "Type" : "Network","TotalTimeSpendInMs" : 97,"ResponseCode" : 200,"Response" : "OK","ServerPerf" : "[b39198b9-e4db-48d2-8dca-7d5f5075b058][ AuthTime::592.784453137096::PostAuthTime::121.465212619722 ][HdfsListStatus :: 00:00:088 ms]%0a[ListStatus :: 00:00:088 ms]%0a[LISTSTATUS :: 00:00:088 ms]%0a","Machine" : "bn4sch103190624","RequestStartTime" : "13/04/16 07:41:11.817","SSLHandshakeCompleteTime" : "13/04/16 07:41:11.817","SSLHandshakeCompleteTimeInMs" : 0,"RequestEndTime" : "13/04/16 07:41:11.914","URL" : "https://xy4.a.net:443/webhdfs/v1/hbase/archive/data?op=LISTSTATUS&user.name=hbase","WebhdfsObjectId" : "a36d695c-52cb-452d-b9ca-853393978413","OAuthTokenRequest" : "13/04/16 07:41:11.600","OAuthTokenResponse" : "13/04/16 07:41:11.600","OAuthRefreshTokenTimeInMs" : 0,"Host" : "zk2-hbase4","LogVersion" : "1.2.2-Dogfood"} 2016-04-13 07:41:11,915 INFO [10.0.0.75,16000,1460531913905_ChoreService_2] web.PrivateAzureDataLakeFileSystem: { "Type" : "API","TotalTimeSpendInMs" : 315,"Method" : "listStatus","jobid" : "NA","WebhdfsObjectId" : "0d02ed5d-c694-497f-9000-dad71994b695","Start" : "13/04/16 07:41:11.600","End" : "13/04/16 07:41:11.915","PathInfo" : "adl://xy4.a.net:443/hbase/archive/data","Result" : "[Lorg.apache.hadoop.fs.FileStatus;@b94a990","LogVersion" : "1.2.2-Dogfood"} 2016-04-13 07:41:11,951 INFO [pool-4330-thread-1] adal4j.UserDiscoveryRequest: [Correlation ID: null] Sent (null) Correlation Id is not same as received (6d531412-4b2a-4d91-9188-f5efd2db49ec). 2016-04-13 07:41:11,952 INFO [pool-4330-thread-1] adal4j.AuthenticationAuthority: [Correlation ID: 08976398-f089-4a6f-a577-881070dea4c6] Instance discovery was successful 2016-04-13 07:41:12,258 INFO [10.0.0.75,16000,1460531913905_ChoreService_2] web.PrivateAzureDataLakeFileSystem: { "Type" : "Network","TotalTimeSpendInMs" : 97,"ResponseCode" : 200,"Response" : "OK","ServerPerf" : "[1e271888-b8de-43f2-8521-7642841ec832][ AuthTime::574.394084810163::PostAuthTime::123.603790402187 ][HdfsListStatus :: 00:00:086 ms]%0a[ListStatus :: 00:00:086 ms]%0a[LISTSTATUS :: 00:00:086 ms]%0a","Machine" : "bn4sch103200823","RequestStartTime" : "13/04/16 07:41:12.161","SSLHandshakeCompleteTime" : "13/04/16 07:41:12.161","SSLHandshakeCompleteTimeInMs" : 0,"RequestEndTime" : "13/04/16 07:41:12.258","URL" : "https://xy4.a.net:443/webhdfs/v1/hbase/archive/data/default?op=LISTSTATUS&user.name=hbase","WebhdfsObjectId" : "a36d695c-52cb-452d-b9ca-853393978413","OAuthTokenRequest" : "13/04/16 07:41:11.915","OAuthTokenResponse" : "13/04/16 07:41:11.915","OAuthRefreshTokenTimeInMs" : 0,"Host" : "zk2-hbase4","LogVersion" : "1.2.2-Dogfood"} 2016-04-13 07:41:12,258 INFO [10.0.0.75,16000,1460531913905_ChoreService_2] web.PrivateAzureDataLakeFileSystem: { "Type" : "API","TotalTimeSpendInMs" : 343,"Method" : "listStatus","jobid" : "NA","WebhdfsObjectId" : "0d02ed5d-c694-497f-9000-dad71994b695","Start" : "13/04/16 07:41:11.915","End" : "13/04/16 07:41:12.258","PathInfo" : "adl://xy4.a.net:443/hbase/archive/data/default","Result" : "[Lorg.apache.hadoop.fs.FileStatus;@66764fc6","LogVersion" : "1.2.2-Dogfood"} 2016-04-13 07:41:12,288 INFO [MASTER_TABLE_OPERATIONS-10.0.0.75:16000-0] web.PrivateAzureDataLakeFileSystem: { "Type" : "Network","TotalTimeSpendInMs" : 272,"ResponseCode" : 200,"Response" : "OK","ServerPerf" : "[2f44361c-0c43-44e4-be42-3523e3da1008][ AuthTime::533.334930059133::PostAuthTime::115.04979646023 ][FsDelete :: 00:00:265 ms]%0a[CleanCacheDelete :: 00:00:000 ms]%0a[Delete : Recursive :: 00:00:266 ms]%0a[DELETE :: 00:00:266 ms]%0a","Machine" : "bn4sch103190624","RequestStartTime" : "13/04/16 07:41:12.015","SSLHandshakeCompleteTime" : "13/04/16 07:41:12.015","SSLHandshakeCompleteTimeInMs" : 0,"RequestEndTime" : "13/04/16 07:41:12.287","URL" : "https://xy4.a.net:443/webhdfs/v1/hbase/.hbase-snapshot/.tmp/snapshot_tb1_create?op=DELETE&user.name=hbase&recursive=true","WebhdfsObjectId" : "a36d695c-52cb-452d-b9ca-853393978413","OAuthTokenRequest" : "13/04/16 07:41:11.803","OAuthTokenResponse" : "13/04/16 07:41:11.804","OAuthRefreshTokenTimeInMs" : 1,"Host" : "zk2-hbase4","LogVersion" : "1.2.2-Dogfood"} 2016-04-13 07:41:12,288 INFO [MASTER_TABLE_OPERATIONS-10.0.0.75:16000-0] web.PrivateAzureDataLakeFileSystem: { "Type" : "API","TotalTimeSpendInMs" : 485,"Method" : "delete","jobid" : "NA","WebhdfsObjectId" : "0d02ed5d-c694-497f-9000-dad71994b695","Start" : "13/04/16 07:41:11.803","End" : "13/04/16 07:41:12.288","PathInfo" : "adl://xy4.a.net:443/hbase/.hbase-snapshot/.tmp/snapshot_tb1_create","Result" : "true","LogVersion" : "1.2.2-Dogfood"} 2016-04-13 07:41:12,306 INFO [pool-4331-thread-1] adal4j.UserDiscoveryRequest: [Correlation ID: null] Sent (null) Correlation Id is not same as received (4b96dbc3-8108-440d-a703-b5a129477838). 2016-04-13 07:41:12,307 INFO [pool-4331-thread-1] adal4j.AuthenticationAuthority: [Correlation ID: 3bb129fa-51f8-4a69-8179-aaf3c39bccac] Instance discovery was successful 2016-04-13 07:41:12,307 DEBUG [MASTER_TABLE_OPERATIONS-10.0.0.75:16000-0] lock.ZKInterProcessLockBase: Released /hbase-unsecure/table-lock/tb1_create/write-master:160000000000001 2016-04-13 07:41:12,308 ERROR [MASTER_TABLE_OPERATIONS-10.0.0.75:16000-0] executor.EventHandler: Caught throwable while processing event C_M_SNAPSHOT_TABLE java.lang.NoClassDefFoundError: Could not initialize class org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos at org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos$SnapshotDataManifest.internalGetFieldAccessorTable(SnapshotProtos.java:3883) at com.google.protobuf.GeneratedMessage.getDescriptorForType(GeneratedMessage.java:98) at com.google.protobuf.AbstractMessage$Builder.findMissingFields(AbstractMessage.java:789) at com.google.protobuf.AbstractMessage$Builder.findMissingFields(AbstractMessage.java:780) at com.google.protobuf.AbstractMessage$Builder.newUninitializedMessageException(AbstractMessage.java:770) at com.google.protobuf.AbstractMessage.newUninitializedMessageException(AbstractMessage.java:237) at com.google.protobuf.AbstractParser.newUninitializedMessageException(AbstractParser.java:57) at com.google.protobuf.AbstractParser.checkMessageInitialized(AbstractParser.java:71) at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:217) at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:223) at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:49) at org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos$SnapshotDataManifest.parseFrom(SnapshotProtos.java:4094) at org.apache.hadoop.hbase.snapshot.SnapshotManifest.readDataManifest(SnapshotManifest.java:433) at org.apache.hadoop.hbase.snapshot.SnapshotManifest.load(SnapshotManifest.java:273) at org.apache.hadoop.hbase.snapshot.SnapshotManifest.open(SnapshotManifest.java:119) at org.apache.hadoop.hbase.master.snapshot.MasterSnapshotVerifier.verifySnapshot(MasterSnapshotVerifier.java:108) at org.apache.hadoop.hbase.master.snapshot.TakeSnapshotHandler.process(TakeSnapshotHandler.java:200) at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:128) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745)