Major Description
when use indextool with -snapshot option and if the mapreduce create multi mapper.this will cause the hdfs file not found or hdfs file exist exception。finally the mapreduce task must be failed. because the mapper use the same restore work dir.
Error: java.io.IOException: java.util.concurrent.ExecutionException: java.io.IOException: The specified region already exists on disk: hdfs://m12v1.mlamp.cn:8020/tmp/index-snapshot-dir/restore-dir/e738c85b-2394-43fc-b9de-b8280bc329ca/data/default/SCOPA.CETUS_EVENT_ZY_SCOPA_31_0516_TRAIN_EVENT/2ab2c1d73d2e31bb5a5e2b394da564f8 at org.apache.hadoop.hbase.util.ModifyRegionUtils.createRegions(ModifyRegionUtils.java:186) at org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper.cloneHdfsRegions(RestoreSnapshotHelper.java:578) at org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper.restoreHdfsRegions(RestoreSnapshotHelper.java:249) at org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper.restoreHdfsRegions(RestoreSnapshotHelper.java:171) at org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper.copySnapshotForScanner(RestoreSnapshotHelper.java:814) at org.apache.phoenix.iterate.TableSnapshotResultIterator.init(TableSnapshotResultIterator.java:77) at org.apache.phoenix.iterate.TableSnapshotResultIterator.<init>(TableSnapshotResultIterator.java:73) at org.apache.phoenix.mapreduce.PhoenixRecordReader.initialize(PhoenixRecordReader.java:126) at org.apache.hadoop.mapred.MapTask$NewTrackingRecordReader.initialize(MapTask.java:548) at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:786) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1709) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158) Caused by: java.util.concurrent.ExecutionException: java.io.IOException: The specified region already exists on disk: hdfs://m12v1.mlamp.cn:8020/tmp/index-snapshot-dir/restore-dir/e738c85b-2394-43fc-b9de-b8280bc329ca/data/default/SCOPA.CETUS_EVENT_ZY_SCOPA_31_0516_TRAIN_EVENT/2ab2c1d73d2e31bb5a5e2b394da564f8 at java.util.concurrent.FutureTask.report(FutureTask.java:122) at java.util.concurrent.FutureTask.get(FutureTask.java:188) at org.apache.hadoop.hbase.util.ModifyRegionUtils.createRegions(ModifyRegionUtils.java:180) ... 15 more Caused by: java.io.IOException: The specified region already exists on disk: hdfs://m12v1.mlamp.cn:8020/tmp/index-snapshot-dir/restore-dir/e738c85b-2394-43fc-b9de-b8280bc329ca/data/default/SCOPA.CETUS_EVENT_ZY_SCOPA_31_0516_TRAIN_EVENT/2ab2c1d73d2e31bb5a5e2b394da564f8 at org.apache.hadoop.hbase.regionserver.HRegionFileSystem.createRegionOnFileSystem(HRegionFileSystem.java:877) at org.apache.hadoop.hbase.regionserver.HRegion.createHRegion(HRegion.java:6252) at org.apache.hadoop.hbase.util.ModifyRegionUtils.createRegion(ModifyRegionUtils.java:205) at org.apache.hadoop.hbase.util.ModifyRegionUtils$1.call(ModifyRegionUtils.java:173) at org.apache.hadoop.hbase.util.ModifyRegionUtils$1.call(ModifyRegionUtils.java:170) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) 2018-06-28 15:01:55 70909 [main] INFO org.apache.hadoop.mapreduce.Job - Task Id : attempt_1530004808977_0011_m_000001_0, Status : FAILED Error: java.io.IOException: java.util.concurrent.ExecutionException: java.io.IOException: The specified region already exists on disk: hdfs://m12v1.mlamp.cn:8020/tmp/index-snapshot-dir/restore-dir/e738c85b-2394-43fc-b9de-b8280bc329ca/data/default/SCOPA.CETUS_EVENT_ZY_SCOPA_31_0516_TRAIN_EVENT/2ab2c1d73d2e31bb5a5e2b394da564f8 at org.apache.hadoop.hbase.util.ModifyRegionUtils.createRegions(ModifyRegionUtils.java:186) at org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper.cloneHdfsRegions(RestoreSnapshotHelper.java:578) at org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper.restoreHdfsRegions(RestoreSnapshotHelper.java:249) at org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper.restoreHdfsRegions(RestoreSnapshotHelper.java:171) at org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper.copySnapshotForScanner(RestoreSnapshotHelper.java:814) at org.apache.phoenix.iterate.TableSnapshotResultIterator.init(TableSnapshotResultIterator.java:77) at org.apache.phoenix.iterate.TableSnapshotResultIterator.<init>(TableSnapshotResultIterator.java:73) at org.apache.phoenix.mapreduce.PhoenixRecordReader.initialize(PhoenixRecordReader.java:126) at org.apache.hadoop.mapred.MapTask$NewTrackingRecordReader.initialize(MapTask.java:548) at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:786) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1709) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158) Caused by: java.util.concurrent.ExecutionException: java.io.IOException: The specified region already exists on disk: hdfs://m12v1.mlamp.cn:8020/tmp/index-snapshot-dir/restore-dir/e738c85b-2394-43fc-b9de-b8280bc329ca/data/default/SCOPA.CETUS_EVENT_ZY_SCOPA_31_0516_TRAIN_EVENT/2ab2c1d73d2e31bb5a5e2b394da564f8 at java.util.concurrent.FutureTask.report(FutureTask.java:122) at java.util.concurrent.FutureTask.get(FutureTask.java:188) at org.apache.hadoop.hbase.util.ModifyRegionUtils.createRegions(ModifyRegionUtils.java:180) ... 15 more Caused by: java.io.IOException: The specified region already exists on disk: hdfs://m12v1.mlamp.cn:8020/tmp/index-snapshot-dir/restore-dir/e738c85b-2394-43fc-b9de-b8280bc329ca/data/default/SCOPA.CETUS_EVENT_ZY_SCOPA_31_0516_TRAIN_EVENT/2ab2c1d73d2e31bb5a5e2b394da564f8 at org.apache.hadoop.hbase.regionserver.HRegionFileSystem.createRegionOnFileSystem(HRegionFileSystem.java:877) at org.apache.hadoop.hbase.regionserver.HRegion.createHRegion(HRegion.java:6252) at org.apache.hadoop.hbase.util.ModifyRegionUtils.createRegion(ModifyRegionUtils.java:205) at org.apache.hadoop.hbase.util.ModifyRegionUtils$1.call(ModifyRegionUtils.java:173) at org.apache.hadoop.hbase.util.ModifyRegionUtils$1.call(ModifyRegionUtils.java:170) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) 2018-06-28 15:01:55 70946 [main] INFO org.apache.hadoop.mapreduce.Job - Task Id : attempt_1530004808977_0011_m_000003_0, Status : FAILED Error: java.io.IOException: java.util.concurrent.ExecutionException: java.io.IOException: The specified region already exists on disk: hdfs://m12v1.mlamp.cn:8020/tmp/index-snapshot-dir/restore-dir/e738c85b-2394-43fc-b9de-b8280bc329ca/data/default/SCOPA.CETUS_EVENT_ZY_SCOPA_31_0516_TRAIN_EVENT/6032d65488e6e7eba1255c20c1047cdc at org.apache.hadoop.hbase.util.ModifyRegionUtils.createRegions(ModifyRegionUtils.java:186) at org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper.cloneHdfsRegions(RestoreSnapshotHelper.java:578) at org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper.restoreHdfsRegions(RestoreSnapshotHelper.java:249) at org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper.restoreHdfsRegions(RestoreSnapshotHelper.java:171) at org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper.copySnapshotForScanner(RestoreSnapshotHelper.java:814) at org.apache.phoenix.iterate.TableSnapshotResultIterator.init(TableSnapshotResultIterator.java:77) at org.apache.phoenix.iterate.TableSnapshotResultIterator.<init>(TableSnapshotResultIterator.java:73) at org.apache.phoenix.mapreduce.PhoenixRecordReader.initialize(PhoenixRecordReader.java:126) at org.apache.hadoop.mapred.MapTask$NewTrackingRecordReader.initialize(MapTask.java:548) at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:786) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1709) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158) Caused by: java.util.concurrent.ExecutionException: java.io.IOException: The specified region already exists on disk: hdfs://m12v1.mlamp.cn:8020/tmp/index-snapshot-dir/restore-dir/e738c85b-2394-43fc-b9de-b8280bc329ca/data/default/SCOPA.CETUS_EVENT_ZY_SCOPA_31_0516_TRAIN_EVENT/6032d65488e6e7eba1255c20c1047cdc at java.util.concurrent.FutureTask.report(FutureTask.java:122) at java.util.concurrent.FutureTask.get(FutureTask.java:188) at org.apache.hadoop.hbase.util.ModifyRegionUtils.createRegions(ModifyRegionUtils.java:180) ... 15 more Caused by: java.io.IOException: The specified region already exists on disk: hdfs://m12v1.mlamp.cn:8020/tmp/index-snapshot-dir/restore-dir/e738c85b-2394-43fc-b9de-b8280bc329ca/data/default/SCOPA.CETUS_EVENT_ZY_SCOPA_31_0516_TRAIN_EVENT/6032d65488e6e7eba1255c20c1047cdc at org.apache.hadoop.hbase.regionserver.HRegionFileSystem.createRegionOnFileSystem(HRegionFileSystem.java:877) at org.apache.hadoop.hbase.regionserver.HRegion.createHRegion(HRegion.java:6252) at org.apache.hadoop.hbase.util.ModifyRegionUtils.createRegion(ModifyRegionUtils.java:205) at org.apache.hadoop.hbase.util.ModifyRegionUtils$1.call(ModifyRegionUtils.java:173) at org.apache.hadoop.hbase.util.ModifyRegionUtils$1.call(ModifyRegionUtils.java:170) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) 2018-06-28 15:01:56 71957 [main] INFO org.apache.hadoop.mapreduce.Job - Task Id : attempt_1530004808977_0011_m_000000_0, Status : FAILED Error: java.lang.NullPointerException at org.apache.phoenix.iterate.TableSnapshotResultIterator.init(TableSnapshotResultIterator.java:82) at org.apache.phoenix.iterate.TableSnapshotResultIterator.<init>(TableSnapshotResultIterator.java:73) at org.apache.phoenix.mapreduce.PhoenixRecordReader.initialize(PhoenixRecordReader.java:126) at org.apache.hadoop.mapred.MapTask$NewTrackingRecordReader.initialize(MapTask.java:548) at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:786) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1709) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)
Attachments
Issue Links
- links to
