diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java index ac87402..270cb26 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java @@ -52,7 +52,8 @@ import org.apache.commons.logging.LogFactory; public class Export { private static final Log LOG = LogFactory.getLog(Export.class); final static String NAME = "export"; - final static String RAW_SCAN="hbase.mapreduce.include.deleted.rows"; + final static String RAW_SCAN = "hbase.mapreduce.include.deleted.rows"; + final static String EXPORT_BATCHING = "hbase.export.scanner.batch"; /** * Mapper. @@ -134,6 +135,15 @@ public class Export { LOG.info("Setting Scan Filter for Export."); s.setFilter(exportFilter); } + + int batching = conf.getInt(EXPORT_BATCHING, -1); + if (batching != -1){ + try{ + s.setBatch(batching); + } catch (RuntimeException e) { + LOG.error("Batching could not be set", e); + } + } LOG.info("versions=" + versions + ", starttime=" + startTime + ", endtime=" + endTime + ", keepDeletedCells=" + raw); return s; @@ -174,6 +184,8 @@ public class Export { + " -Dhbase.client.scanner.caching=100\n" + " -Dmapred.map.tasks.speculative.execution=false\n" + " -Dmapred.reduce.tasks.speculative.execution=false"); + System.err.println("For tables with very wide rows consider setting the batch size as below:\n" + + " -D" + EXPORT_BATCHING + "=10"); } /** @@ -192,4 +204,4 @@ public class Export { Job job = createSubmittableJob(conf, otherArgs); System.exit(job.waitForCompletion(true)? 0 : 1); } -} \ No newline at end of file +} diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java index bc05754..c629141 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java @@ -57,6 +57,7 @@ public class TestImportExport { private static final byte[] FAMILYB = Bytes.toBytes(FAMILYB_STRING); private static final byte[] QUAL = Bytes.toBytes("q"); private static final String OUTPUT_DIR = "outputdir"; + private static final String EXPORT_BATCHING = "100"; private static MiniHBaseCluster cluster; private static long now = System.currentTimeMillis(); @@ -102,6 +103,7 @@ public class TestImportExport { String[] args = new String[] { EXPORT_TABLE, OUTPUT_DIR, + EXPORT_BATCHING, "1000" }; @@ -171,6 +173,7 @@ public class TestImportExport { "-D" + Export.RAW_SCAN + "=true", EXPORT_TABLE, OUTPUT_DIR, + EXPORT_BATCHING, "1000" };