From 413c53d54fd8d3f3915a73ed6f4121e1cf4e9d7c Mon Sep 17 00:00:00 2001 From: sunyerui Date: Tue, 2 Aug 2016 23:27:57 +0800 Subject: [PATCH] KYLIN-1934 'Value not exist' During Cube Merging Caused by Empty Dict --- .../dict/MultipleDictionaryValueEnumerator.java | 22 +-- .../MultipleDictionaryValueEnumeratorTest.java | 149 +++++++++++++++++++++ 2 files changed, 161 insertions(+), 10 deletions(-) create mode 100644 core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java index df7b1c6..e9010e0 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java @@ -55,19 +55,21 @@ public class MultipleDictionaryValueEnumerator implements IDictionaryValueEnumer @Override public boolean moveNext() throws IOException { - if (curDictIndex < dictionaryList.size() && curKey <= curDict.getMaxId()) { - byte[] buffer = new byte[curDict.getSizeOfValue()]; - int size = curDict.getValueBytesFromId(curKey, buffer, 0); - curValue = Bytes.copy(buffer, 0, size); + while (curDictIndex < dictionaryList.size()) { + if (curKey <= curDict.getMaxId()) { + byte[] buffer = new byte[curDict.getSizeOfValue()]; + int size = curDict.getValueBytesFromId(curKey, buffer, 0); + curValue = Bytes.copy(buffer, 0, size); + curKey ++; - if (++curKey > curDict.getMaxId()) { - if (++curDictIndex < dictionaryList.size()) { - curDict = dictionaryList.get(curDictIndex); - curKey = curDict.getMinId(); - } + return true; } - return true; + // move to next dict if exists + if (++curDictIndex < dictionaryList.size()) { + curDict = dictionaryList.get(curDictIndex); + curKey = curDict.getMinId(); + } } curValue = null; return false; diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java new file mode 100644 index 0000000..6e0f88a --- /dev/null +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java @@ -0,0 +1,149 @@ +package org.apache.kylin.dict; + +import org.apache.kylin.common.util.Bytes; +import org.apache.kylin.common.util.Dictionary; +import org.junit.Test; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertArrayEquals; + +/** + * Created by sunyerui on 16/8/2. + */ +public class MultipleDictionaryValueEnumeratorTest { + + private static DictionaryInfo createDictInfo(int[] values) { + MockDictionary mockDict = new MockDictionary(); + mockDict.values = values; + DictionaryInfo info = new DictionaryInfo(); + info.setDictionaryObject(mockDict); + return info; + } + + private static Integer[] enumerateDictInfoList(List dictionaryInfoList) throws IOException { + MultipleDictionaryValueEnumerator enumerator = new MultipleDictionaryValueEnumerator(dictionaryInfoList); + List values = new ArrayList<>(); + while (enumerator.moveNext()) { + values.add(Bytes.toInt(enumerator.current())); + } + return values.toArray(new Integer[0]); + } + + @Test + public void testNormalDicts() throws IOException { + List dictionaryInfoList = new ArrayList<>(2); + dictionaryInfoList.add(createDictInfo(new int[]{0, 1, 2})); + dictionaryInfoList.add(createDictInfo(new int[]{4, 5, 6})); + + Integer[] values = enumerateDictInfoList(dictionaryInfoList); + assertEquals(6, values.length); + assertArrayEquals(new Integer[]{0, 1, 2, 4, 5, 6}, values); + } + + @Test + public void testFirstEmptyDicts() throws IOException { + List dictionaryInfoList = new ArrayList<>(2); + dictionaryInfoList.add(createDictInfo(new int[]{})); + dictionaryInfoList.add(createDictInfo(new int[]{4, 5, 6})); + + Integer[] values = enumerateDictInfoList(dictionaryInfoList); + assertEquals(3, values.length); + assertArrayEquals(new Integer[]{4, 5, 6}, values); + } + + @Test + public void testMiddleEmptyDicts() throws IOException { + List dictionaryInfoList = new ArrayList<>(3); + dictionaryInfoList.add(createDictInfo(new int[]{0, 1, 2})); + dictionaryInfoList.add(createDictInfo(new int[]{})); + dictionaryInfoList.add(createDictInfo(new int[]{7, 8, 9})); + + Integer[] values = enumerateDictInfoList(dictionaryInfoList); + assertEquals(6, values.length); + assertArrayEquals(new Integer[]{0, 1, 2, 7, 8, 9}, values); + } + + @Test + public void testLastEmptyDicts() throws IOException { + List dictionaryInfoList = new ArrayList<>(3); + dictionaryInfoList.add(createDictInfo(new int[]{0, 1, 2})); + dictionaryInfoList.add(createDictInfo(new int[]{6, 7, 8})); + dictionaryInfoList.add(createDictInfo(new int[]{})); + + Integer[] values = enumerateDictInfoList(dictionaryInfoList); + assertEquals(6, values.length); + assertArrayEquals(new Integer[]{0, 1, 2, 6, 7, 8}, values); + } + + public static class MockDictionary extends Dictionary { + public int[] values; + + @Override + public int getMinId() { + return 0; + } + + @Override + public int getMaxId() { + return values.length-1; + } + + @Override + public int getSizeOfId() { + return 4; + } + + @Override + public int getSizeOfValue() { + return 4; + } + + @Override + protected int getIdFromValueImpl(Object value, int roundingFlag) { + return 0; + } + + @Override + protected Object getValueFromIdImpl(int id) { + return null; + } + + @Override + protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) { + return 0; + } + + @Override + protected byte[] getValueBytesFromIdImpl(int id) { + return null; + } + + @Override + protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) { + System.arraycopy(Bytes.toBytes(values[id]), 0, returnValue, offset, 4); + return 4; + } + + @Override + public void dump(PrintStream out) {} + + @Override + public void write(DataOutput out) throws IOException {} + + @Override + public void readFields(DataInput in) throws IOException {} + + @Override + public boolean contains(Dictionary another) { + return false; + } + } + +} \ No newline at end of file -- 2.3.2 (Apple Git-55)