diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 371cb0f..ad525ae 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -513,23 +513,20 @@ // Maximum fraction of heap that can be used by ORC file writers HIVE_ORC_FILE_MEMORY_POOL("hive.exec.orc.memory.pool", 0.5f), // 50% // Define the version of the file to write - HIVE_ORC_WRITE_FORMAT("hive.exec.orc.write.format", null), + HIVE_ORC_WRITE_FORMAT("hive.exec.orc.write.format", "0.12"), // Define the default ORC stripe size - HIVE_ORC_DEFAULT_STRIPE_SIZE("hive.exec.orc.default.stripe.size", - 256L * 1024 * 1024), + HIVE_ORC_DEFAULT_STRIPE_SIZE("hive.exec.orc.default.stripe.size", 256L * 1024 * 1024), // Define the default ORC index stripe - HIVE_ORC_DEFAULT_ROW_INDEX_STRIDE("hive.exec.orc.default.row.index.stride" - , null), + HIVE_ORC_DEFAULT_ROW_INDEX_STRIDE("hive.exec.orc.default.row.index.stride", 10000), // Define the default ORC buffer size - HIVE_ORC_DEFAULT_BUFFER_SIZE("hive.exec.orc.default.buffer.size", null), + HIVE_ORC_DEFAULT_BUFFER_SIZE("hive.exec.orc.default.buffer.size", 256 * 1024), // Define the default block padding - HIVE_ORC_DEFAULT_BLOCK_PADDING("hive.exec.orc.default.block.padding", - null), + HIVE_ORC_DEFAULT_BLOCK_PADDING("hive.exec.orc.default.block.padding", true), // Define the default orc compress - HIVE_ORC_DEFAULT_COMPRESS("hive.exec.orc.default.compress", null), + HIVE_ORC_DEFAULT_COMPRESS("hive.exec.orc.default.compress", "ZLIB", + new StringsValidator("NONE", "ZLIB", "SNAPPY", "LZO")), - HIVE_ORC_DICTIONARY_KEY_SIZE_THRESHOLD( - "hive.exec.orc.dictionary.key.size.threshold", 0.8f), + HIVE_ORC_DICTIONARY_KEY_SIZE_THRESHOLD("hive.exec.orc.dictionary.key.size.threshold", 0.8f), HIVE_ORC_INCLUDE_FILE_FOOTER_IN_SPLITS("hive.orc.splits.include.file.footer", false), HIVE_ORC_CACHE_STRIPE_DETAILS_SIZE("hive.orc.cache.stripe.details.size", 10000), diff --git conf/hive-default.xml.template conf/hive-default.xml.template index 66d22f9..a14421b 100644 --- conf/hive-default.xml.template +++ conf/hive-default.xml.template @@ -1772,6 +1772,62 @@ + hive.exec.orc.memory.pool + 0.5 + + Maximum fraction of heap that can be used by ORC file writers. + + + + + hive.exec.orc.write.format + 0.12 + + Define the version of the ORC file to write. + + + + + hive.exec.orc.default.stripe.size + 268435456 + + Define the default ORC stripe size. + + + + + hive.exec.orc.default.row.index.stride + 10000 + + Define the default ORC index stripe. + + + + + hive.exec.orc.default.buffer.size + 262144 + + Define the default ORC buffer size. + + + + + hive.exec.orc.default.block.padding + true + + Define the default block padding. + + + + + hive.exec.orc.default.compress + ZLIB + + Define the default compression codec for ORC file. + + + + hive.exec.orc.dictionary.key.size.threshold 0.8 diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java index e864473..7a0afba 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java @@ -104,14 +104,6 @@ public int getMinor() { public static final String ENABLE_INDEXES = "orc.create.index"; public static final String BLOCK_PADDING = "orc.block.padding"; - static final long DEFAULT_STRIPE_SIZE = - HiveConf.ConfVars.HIVE_ORC_DEFAULT_STRIPE_SIZE.defaultLongVal; - static final CompressionKind DEFAULT_COMPRESSION_KIND = - CompressionKind.ZLIB; - static final int DEFAULT_BUFFER_SIZE = 256 * 1024; - static final int DEFAULT_ROW_INDEX_STRIDE = 10000; - static final boolean DEFAULT_BLOCK_PADDING = true; - // unused private OrcFile() {} @@ -140,10 +132,10 @@ public static Reader createReader(FileSystem fs, Path path, FileMetaInfo fileMet private FileSystem fileSystemValue = null; private ObjectInspector inspectorValue = null; private long stripeSizeValue; - private int rowIndexStrideValue = DEFAULT_ROW_INDEX_STRIDE; - private int bufferSizeValue = DEFAULT_BUFFER_SIZE; - private boolean blockPaddingValue = DEFAULT_BLOCK_PADDING; - private CompressionKind compressValue = DEFAULT_COMPRESSION_KIND; + private int rowIndexStrideValue; + private int bufferSizeValue; + private boolean blockPaddingValue; + private CompressionKind compressValue; private MemoryManager memoryManagerValue; private Version versionValue; @@ -151,28 +143,17 @@ public static Reader createReader(FileSystem fs, Path path, FileMetaInfo fileMet configuration = conf; memoryManagerValue = getMemoryManager(conf); stripeSizeValue = - conf.getLong(HiveConf.ConfVars.HIVE_ORC_DEFAULT_STRIPE_SIZE.varname, - DEFAULT_STRIPE_SIZE); + HiveConf.getLongVar(conf, HiveConf.ConfVars.HIVE_ORC_DEFAULT_STRIPE_SIZE); rowIndexStrideValue = - conf.getInt(HiveConf.ConfVars.HIVE_ORC_DEFAULT_ROW_INDEX_STRIDE - .varname, DEFAULT_ROW_INDEX_STRIDE); + HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_ORC_DEFAULT_ROW_INDEX_STRIDE); bufferSizeValue = - conf.getInt(HiveConf.ConfVars.HIVE_ORC_DEFAULT_BUFFER_SIZE.varname, - DEFAULT_ROW_INDEX_STRIDE); + HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_ORC_DEFAULT_BUFFER_SIZE); blockPaddingValue = - conf.getBoolean(HiveConf.ConfVars.HIVE_ORC_DEFAULT_BLOCK_PADDING - .varname, DEFAULT_BLOCK_PADDING); - compressValue = - CompressionKind.valueOf(conf.get(HiveConf.ConfVars - .HIVE_ORC_DEFAULT_COMPRESS.varname, - DEFAULT_COMPRESSION_KIND.toString())); - String versionName = - conf.get(HiveConf.ConfVars.HIVE_ORC_WRITE_FORMAT.varname); - if (versionName == null) { - versionValue = Version.CURRENT; - } else { - versionValue = Version.byName(versionName); - } + HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_ORC_DEFAULT_BLOCK_PADDING); + compressValue = CompressionKind.valueOf( + HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_ORC_DEFAULT_COMPRESS)); + versionValue = Version.byName( + HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_ORC_WRITE_FORMAT)); } /**