diff --git ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFileStripeMergeRecordReader.java ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFileStripeMergeRecordReader.java new file mode 100644 index 0000000..e2257fd --- /dev/null +++ ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFileStripeMergeRecordReader.java @@ -0,0 +1,121 @@ +/* + * Copyright 2015 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import java.io.File; +import java.io.IOException; +import java.util.Random; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.FileSplit; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; + +public class TestOrcFileStripeMergeRecordReader { + + private final int DEFAULT_STRIPE_SIZE = 5000; + + private OrcFileKeyWrapper key; + private OrcFileValueWrapper value; + private Path tmpPath; + private Configuration conf; + private FileSystem fs; + + @Rule + public TestName testCaseName = new TestName(); + + @Before + public void setup() throws IOException { + conf = new Configuration(); + fs = FileSystem.getLocal(conf); + key = new OrcFileKeyWrapper(); + value = new OrcFileValueWrapper(); + tmpPath = prepareTmpPath(); + } + + @Test + public void testSplitStartsWithOffset() throws IOException { + createOrcFile(DEFAULT_STRIPE_SIZE, DEFAULT_STRIPE_SIZE + 1); + FileStatus fileStatus = fs.getFileStatus(tmpPath); + long length = fileStatus.getLen(); + long offset = length / 2; + FileSplit split = new FileSplit(tmpPath, offset, length, (String[])null); + OrcFileStripeMergeRecordReader reader = new OrcFileStripeMergeRecordReader(conf, split); + reader.next(key, value); + Assert.assertEquals("InputPath", tmpPath, key.getInputPath()); + Assert.assertEquals("NumberOfValues", 1L, value.getStripeStatistics().getColStats(0).getNumberOfValues()); + reader.close(); + } + + private void createOrcFile(int stripSize, int numberOfRows) throws IOException { + ObjectInspector inspector; + synchronized (TestOrcFileStripeMergeRecordReader.class) { + inspector = ObjectInspectorFactory.getReflectionObjectInspector + (StringIntIntIntRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); + } + + Writer writer = OrcFile.createWriter(tmpPath, + OrcFile.writerOptions(conf) + .inspector(inspector) + .stripeSize(stripSize) + .compress(CompressionKind.ZLIB) + .bufferSize(5000) + .rowIndexStride(1000)); + + Random rand = new Random(157); + + for (int i = 0; i < numberOfRows; i++) { + writer.addRow(new StringIntIntIntRow( + Integer.toBinaryString(i), + rand.nextInt(), + rand.nextInt(), + rand.nextInt() + )); + } + writer.close(); + } + + private Path prepareTmpPath() throws IOException { + Path path = new Path(System.getProperty("test.tmp.dir", + "target" + File.separator + "test" + File.separator + "tmp") + + File.separator + "TestOrcFile." + testCaseName.getMethodName() + ".orc"); + fs.delete(path, false); + return path; + } + + public static class StringIntIntIntRow { + + Text string1 = new Text(); + int int1; + int int2; + int int3; + + StringIntIntIntRow(String string1, int int1, int int2, int int3) { + this.string1.set(string1); + this.int1 = int1; + this.int2 = int2; + this.int3 = int3; + } + } +}