From 6372c2ecadde988e21ada21d707ffea2a017d7f1 Mon Sep 17 00:00:00 2001 From: chenyechao Date: Wed, 30 Jan 2019 20:05:31 +0800 Subject: [PATCH] HBASE-21810 bulkload support set hfile compression on client --- .../hadoop/hbase/mapreduce/HFileOutputFormat2.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java index 31e7e5a770..3f8c8a1d2d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java @@ -112,9 +112,11 @@ public class HFileOutputFormat2 // This constant is public since the client can modify this when setting // up their conf object and thus refer to this symbol. // It is present for backwards compatibility reasons. Use it only to - // override the auto-detection of datablock encoding. + // override the auto-detection of datablock encoding and compression. public static final String DATABLOCK_ENCODING_OVERRIDE_CONF_KEY = "hbase.mapreduce.hfileoutputformat.datablock.encoding"; + public static final String COMPRESSION_OVERRIDE_CONF_KEY = + "hbase.mapreduce.hfileoutputformat.compression"; /** * Keep locality while generating HFiles for bulkload. See HBASE-12596 @@ -150,6 +152,13 @@ public class HFileOutputFormat2 Compression.Algorithm.NONE.getName()); final Algorithm defaultCompression = AbstractHFileWriter .compressionByName(defaultCompressionStr); + String compressionStr = conf.get(COMPRESSION_OVERRIDE_CONF_KEY); + final Algorithm overriddenCompression; + if (compressionStr != null) { + overriddenCompression = Compression.getCompressionAlgorithmByName(compressionStr); + } else { + overriddenCompression = null; + } final boolean compactionExclude = conf.getBoolean( "hbase.mapreduce.hfileoutputformat.compaction.exclude", false); @@ -291,7 +300,8 @@ public class HFileOutputFormat2 throws IOException { WriterLength wl = new WriterLength(); Path familydir = new Path(outputdir, Bytes.toString(family)); - Algorithm compression = compressionMap.get(family); + Algorithm compression = overriddenCompression; + compression = compression == null ? compressionMap.get(family) : compression; compression = compression == null ? defaultCompression : compression; BloomType bloomType = bloomTypeMap.get(family); bloomType = bloomType == null ? BloomType.NONE : bloomType; -- 2.18.0.windows.1