diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java index 552982c..386a53c 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java +++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java @@ -52,6 +52,11 @@ // Proportion of extra space to provide when allocating more buffer space. static final float EXTRA_SPACE_FACTOR = (float) 1.2; + // Do not grow/copy the buffer if the size goes past this value. + // Leave the old buffer as is (the previous rows will reference it), + // and create a new buffer to store the value for the next row(s). + static final int BUFFER_SIZE_THRESHOLD = 1024 * 1024; + /** * Use this constructor for normal operation. * All column vectors should be the default size normally. @@ -243,12 +248,31 @@ public void increaseBufferSpace(int nextElemLength) { int newLength = 2 * buffer.length; while((nextFree + nextElemLength) > newLength) { newLength *= 2; + if (newLength < 0) { + throw new RuntimeException("Overflow of buffer size. buffer.length=" + buffer.length + + ", nextElemLength=" + nextElemLength); + } } - // Allocate new buffer, copy data to it, and set buffer to new buffer. - byte[] newBuffer = new byte[newLength]; - System.arraycopy(buffer, 0, newBuffer, 0, nextFree); - buffer = newBuffer; + if (newLength <= BUFFER_SIZE_THRESHOLD) { + // Allocate new buffer, copy data to it, and set buffer to new buffer. + byte[] newBuffer = new byte[newLength]; + System.arraycopy(buffer, 0, newBuffer, 0, nextFree); + buffer = newBuffer; + } else { + // Do not continue resizing buffer - elements can live in different byte buffers. + // Create a brand new buffer for the next element + newLength = 2 * DEFAULT_BUFFER_SIZE; + while(nextElemLength > newLength) { + newLength *= 2; + if (newLength < 0) { + throw new RuntimeException("Overflow of buffer size. buffer.length=" + buffer.length + + ", nextElemLength=" + nextElemLength); + } + } + buffer = new byte[newLength]; + nextFree = 0; + } } /** Copy the current object contents into the output. Only copy selected entries,