diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java index ac87402..6fb5ad6 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java @@ -52,7 +52,8 @@ import org.apache.commons.logging.LogFactory; public class Export { private static final Log LOG = LogFactory.getLog(Export.class); final static String NAME = "export"; - final static String RAW_SCAN="hbase.mapreduce.include.deleted.rows"; + final static String RAW_SCAN = "hbase.mapreduce.include.deleted.rows"; + final static String EXPORT_BATCHING = "hbase.client.scanner.batch"; /** * Mapper. @@ -134,6 +135,12 @@ public class Export { LOG.info("Setting Scan Filter for Export."); s.setFilter(exportFilter); } + + int batching = conf.getInt(EXPORT_BATCHING, -1); + if (batching != -1){ + s.setBatch(batching); + LOG.info("Setting Batchsize for Export"); + } LOG.info("versions=" + versions + ", starttime=" + startTime + ", endtime=" + endTime + ", keepDeletedCells=" + raw); return s; @@ -174,6 +181,8 @@ public class Export { + " -Dhbase.client.scanner.caching=100\n" + " -Dmapred.map.tasks.speculative.execution=false\n" + " -Dmapred.reduce.tasks.speculative.execution=false"); + System.err.println("For tables with very wide rows consider setting the batch size as below:\n" + + " -D" + EXPORT_BATCHING + "=10"); } /** @@ -192,4 +201,4 @@ public class Export { Job job = createSubmittableJob(conf, otherArgs); System.exit(job.waitForCompletion(true)? 0 : 1); } -} \ No newline at end of file +}