diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java index 5ee8e32..0a60985 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java @@ -33,7 +33,6 @@ import org.apache.hadoop.hive.shims.CombineHiveKey; import java.io.IOException; -import java.util.List; /** * Fast file merge operator for ORC files. @@ -47,7 +46,7 @@ // not be merged. CompressionKind compression = null; long compressBuffSize = 0; - List version; + OrcFile.Version version; int columnCount = 0; int rowIndexStride = 0; @@ -90,13 +89,16 @@ private void processKeyValuePairs(Object key, Object value) if (outWriter == null) { compression = k.getCompression(); compressBuffSize = k.getCompressBufferSize(); - version = k.getVersionList(); + version = k.getVersion(); columnCount = k.getTypes().get(0).getSubtypesCount(); rowIndexStride = k.getRowIndexStride(); - // block size and stripe size will be from config + // create a writer that has same configuration as input orc file outWriter = OrcFile.createWriter(outPath, - OrcFile.writerOptions(jc).compress(compression) + OrcFile.writerOptions(jc) + .compress(compression) + .version(version) + .rowIndexStride(rowIndexStride) .inspector(reader.getObjectInspector())); LOG.info("ORC merge file output path: " + outPath); } @@ -167,7 +169,7 @@ private boolean checkCompatibility(OrcFileKeyWrapper k) { } - if (!k.getVersionList().equals(version)) { + if (!k.getVersion().equals(version)) { LOG.info("Incompatible ORC file merge! Version does not match for " + k.getInputPath()); return false; diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileKeyWrapper.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileKeyWrapper.java index 6eb0e22..11f05c6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileKeyWrapper.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileKeyWrapper.java @@ -18,15 +18,14 @@ package org.apache.hadoop.hive.ql.io.orc; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.WritableComparable; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.util.List; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.ql.io.orc.OrcProto; -import org.apache.hadoop.io.WritableComparable; - /** * Key for OrcFileMergeMapper task. Contains orc file related information that * should match before merging two orc files. @@ -38,14 +37,14 @@ protected long compressBufferSize; protected List types; protected int rowIndexStride; - protected List versionList; + protected OrcFile.Version version; - public List getVersionList() { - return versionList; + public OrcFile.Version getVersion() { + return version; } - public void setVersionList(List versionList) { - this.versionList = versionList; + public void setVersion(OrcFile.Version version) { + this.version = version; } public int getRowIndexStride() { diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java index db5bbb5..603b75a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java @@ -18,16 +18,16 @@ package org.apache.hadoop.hive.ql.io.orc; -import java.io.IOException; -import java.util.Iterator; -import java.util.List; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.FileSplit; import org.apache.hadoop.mapred.RecordReader; +import java.io.IOException; +import java.util.Iterator; +import java.util.List; + public class OrcFileStripeMergeRecordReader implements RecordReader { @@ -88,7 +88,7 @@ protected boolean nextStripe(OrcFileKeyWrapper keyWrapper, OrcFileValueWrapper v keyWrapper.setInputPath(path); keyWrapper.setCompression(reader.getCompression()); keyWrapper.setCompressBufferSize(reader.getCompressionSize()); - keyWrapper.setVersionList(((ReaderImpl) reader).getFileMetaInfo().versionList); + keyWrapper.setVersion(reader.getFileVersion()); keyWrapper.setRowIndexStride(reader.getRowIndexStride()); keyWrapper.setTypes(reader.getTypes()); } else {