diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/CrossMapEqualComparer.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/CrossMapEqualComparer.java new file mode 100644 index 0000000..114dddf --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/CrossMapEqualComparer.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector; + +import java.util.Map; +/* + * The equality is implemented fully, the greater-than/less-than + * values do not implement a transitive relation. + */ + +public class CrossMapEqualComparer implements MapEqualComparer { + @Override + public int compare(Object o1, MapObjectInspector moi1, Object o2, MapObjectInspector moi2) { + int mapsize1 = moi1.getMapSize(o1); + int mapsize2 = moi2.getMapSize(o2); + if (mapsize1 != mapsize2) { + return mapsize1 - mapsize2; + } + ObjectInspector mkoi1 = moi1.getMapKeyObjectInspector(); + ObjectInspector mkoi2 = moi2.getMapKeyObjectInspector(); + + ObjectInspector mvoi1 = moi1.getMapValueObjectInspector(); + ObjectInspector mvoi2 = moi2.getMapValueObjectInspector(); + + Map map1 = moi1.getMap(o1); + Map map2 = moi2.getMap(o2); + for (Object mk1 : map1.keySet()) { + boolean notFound = true; + for (Object mk2 : map2.keySet()) { + int rc = ObjectInspectorUtils.compare(mk1, mkoi1, mk2, mkoi2, this); + if (rc != 0) { + continue; + } + notFound = false; + Object mv1 = map1.get(mk1); + Object mv2 = map2.get(mk2); + rc = ObjectInspectorUtils.compare(mv1, mvoi1, mv2, mvoi2, this); + if (rc != 0) { + return rc; + } else { + break; + } + } + if (notFound) { + return 1; + } + } + return 0; + } + +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/FullMapEqualComparer.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/FullMapEqualComparer.java new file mode 100644 index 0000000..f6169bb --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/FullMapEqualComparer.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector; + +import java.util.Arrays; +import java.util.Comparator; +import java.util.Map; +import java.util.TreeMap; + +/* + * The equality is implemented fully, the implementation sorts the maps + * by their keys to provide a transitive compare. + */ + +public class FullMapEqualComparer implements MapEqualComparer { + + private static class MapKeyComparator implements Comparator { + + private ObjectInspector oi; + + MapKeyComparator(ObjectInspector oi) { + this.oi = oi; + } + + @Override + public int compare(Object o1, Object o2) { + return ObjectInspectorUtils.compare(o1, oi, o2, oi); + } + } + + @Override + public int compare(Object o1, MapObjectInspector moi1, Object o2, MapObjectInspector moi2) { + int mapsize1 = moi1.getMapSize(o1); + int mapsize2 = moi2.getMapSize(o2); + if (mapsize1 != mapsize2) { + return mapsize1 - mapsize2; + } + ObjectInspector mkoi1 = moi1.getMapKeyObjectInspector(); + ObjectInspector mkoi2 = moi2.getMapKeyObjectInspector(); + + ObjectInspector mvoi1 = moi1.getMapValueObjectInspector(); + ObjectInspector mvoi2 = moi2.getMapValueObjectInspector(); + + Map map1 = moi1.getMap(o1); + Map map2 = moi2.getMap(o2); + + Object[] sortedMapKeys1 = map1.keySet().toArray(); + Arrays.sort(sortedMapKeys1, new MapKeyComparator(mkoi1)); + + Object[] sortedMapKeys2 = map2.keySet().toArray(); + Arrays.sort(sortedMapKeys2, new MapKeyComparator(mkoi2)); + + for (int i = 0; i < mapsize1; ++i) { + Object mk1 = sortedMapKeys1[i]; + Object mk2 = sortedMapKeys2[i]; + int rc = ObjectInspectorUtils.compare(mk1, mkoi1, mk2, mkoi2, this); + if (rc != 0) { + return rc; + } + Object mv1 = map1.get(mk1); + Object mv2 = map2.get(mk2); + rc = ObjectInspectorUtils.compare(mv1, mvoi1, mv2, mvoi2, this); + if (rc != 0) { + return rc; + } + } + return 0; + } + +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapEqualComparer.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapEqualComparer.java new file mode 100644 index 0000000..adde408 --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapEqualComparer.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector; + +public interface MapEqualComparer { + /* + * Compare the two map objects for equality. + */ + public int compare(Object o1, MapObjectInspector moi1, + Object o2, MapObjectInspector moi2); +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java index 2b77072..e504dad 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java @@ -515,6 +515,14 @@ public final class ObjectInspectorUtils { */ public static int compare(Object o1, ObjectInspector oi1, Object o2, ObjectInspector oi2) { + return compare(o1, oi1, o2, oi2, null); + } + + /** + * Compare two objects with their respective ObjectInspectors. + */ + public static int compare(Object o1, ObjectInspector oi1, Object o2, + ObjectInspector oi2, MapEqualComparer mapEqualComparer) { if (oi1.getCategory() != oi2.getCategory()) { return oi1.getCategory().compareTo(oi2.getCategory()); } @@ -599,7 +607,8 @@ public final class ObjectInspectorUtils { for (int i = 0; i < minimum; i++) { int r = compare(soi1.getStructFieldData(o1, fields1.get(i)), fields1 .get(i).getFieldObjectInspector(), soi2.getStructFieldData(o2, - fields2.get(i)), fields2.get(i).getFieldObjectInspector()); + fields2.get(i)), fields2.get(i).getFieldObjectInspector(), + mapEqualComparer); if (r != 0) { return r; } @@ -613,7 +622,8 @@ public final class ObjectInspectorUtils { for (int i = 0; i < minimum; i++) { int r = compare(loi1.getListElement(o1, i), loi1 .getListElementObjectInspector(), loi2.getListElement(o2, i), loi2 - .getListElementObjectInspector()); + .getListElementObjectInspector(), + mapEqualComparer); if (r != 0) { return r; } @@ -621,7 +631,11 @@ public final class ObjectInspectorUtils { return loi1.getListLength(o1) - loi2.getListLength(o2); } case MAP: { - throw new RuntimeException("Compare on map type not supported!"); + if (mapEqualComparer == null) { + throw new RuntimeException("Compare on map type not supported!"); + } else { + return mapEqualComparer.compare(o1, (MapObjectInspector)oi1, o2, (MapObjectInspector)oi2); + } } case UNION: { UnionObjectInspector uoi1 = (UnionObjectInspector) oi1; @@ -633,7 +647,8 @@ public final class ObjectInspectorUtils { } return compare(uoi1.getField(o1), uoi1.getObjectInspectors().get(tag1), - uoi2.getField(o2), uoi2.getObjectInspectors().get(tag2)); + uoi2.getField(o2), uoi2.getObjectInspectors().get(tag2), + mapEqualComparer); } default: throw new RuntimeException("Compare on unknown type: " diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SimpleMapEqualComparer.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SimpleMapEqualComparer.java new file mode 100644 index 0000000..286aa24 --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SimpleMapEqualComparer.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector; + +import java.util.Map; + +/* + * Assumes that a getMapValueElement on object2 will work with a key from + * object1. The equality is implemented fully, the greater-than/less-than + * values do not implement a transitive relation. + */ +public class SimpleMapEqualComparer implements MapEqualComparer { + + @Override + public int compare(Object o1, MapObjectInspector moi1, Object o2, MapObjectInspector moi2) { + int mapsize1 = moi1.getMapSize(o1); + int mapsize2 = moi2.getMapSize(o2); + if (mapsize1 != mapsize2) { + return mapsize1 - mapsize2; + } + ObjectInspector mvoi1 = moi1.getMapValueObjectInspector(); + ObjectInspector mvoi2 = moi2.getMapValueObjectInspector(); + Map map1 = moi1.getMap(o1); + for (Object mk1: map1.keySet()) { + int rc = ObjectInspectorUtils.compare(moi1.getMapValueElement(o1, mk1), mvoi1, + moi2.getMapValueElement(o2, mk1), mvoi2, this); + if (rc != 0) { + return rc; + } + } + return 0; + } +} diff --git serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestCrossMapEqualComparer.java serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestCrossMapEqualComparer.java new file mode 100644 index 0000000..8be5da4 --- /dev/null +++ serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestCrossMapEqualComparer.java @@ -0,0 +1,179 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector; + +import java.io.IOException; +import java.util.Map; +import java.util.Properties; +import java.util.TreeMap; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde.Constants; +import org.apache.hadoop.hive.serde2.ByteStream; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; +import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions; +import org.apache.hadoop.io.Text; + +import junit.framework.TestCase; + +public class TestCrossMapEqualComparer extends TestCase { + + public static class IntegerStringMapHolder { + Map mMap; + + public IntegerStringMapHolder() { + mMap = new TreeMap(); + } + } + + public void testSameType() { + // empty maps + IntegerStringMapHolder o1 = new IntegerStringMapHolder(); + IntegerStringMapHolder o2 = new IntegerStringMapHolder(); + ObjectInspector oi1 = ObjectInspectorFactory + .getReflectionObjectInspector(IntegerStringMapHolder.class, ObjectInspectorOptions.JAVA); + int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi1, new CrossMapEqualComparer()); + assertEquals(0, rc); + + // equal maps + o1.mMap.put(42, "The answer to Life, Universe And Everything"); + o2.mMap.put(42, "The answer to Life, Universe And Everything"); + + o1.mMap.put(1729, "A taxi cab number"); + o2.mMap.put(1729, "A taxi cab number"); + rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi1, new CrossMapEqualComparer()); + assertEquals(0, rc); + + // unequal maps + o2.mMap.put(1729, "Hardy-Ramanujan Number"); + rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi1, new CrossMapEqualComparer()); + assertFalse(0 == rc); + } + + public static class TextStringMapHolder { + Map mMap; + + public TextStringMapHolder() { + mMap = new TreeMap(); + } + } + + Object serializeAndDeserialize(TextStringMapHolder o1, StructObjectInspector oi1, + LazySimpleSerDe serde, + SerDeParameters serdeParams) throws IOException, SerDeException { + ByteStream.Output serializeStream = new ByteStream.Output(); + LazySimpleSerDe.serialize(serializeStream, o1, oi1, serdeParams + .getSeparators(), 0, serdeParams.getNullSequence(), serdeParams + .isEscaped(), serdeParams.getEscapeChar(), serdeParams + .getNeedsEscape()); + Text t = new Text(serializeStream.toByteArray()); + return serde.deserialize(t); + } + + public void testCompatibleType() throws SerDeException, IOException { + // empty maps + TextStringMapHolder o1 = new TextStringMapHolder(); + StructObjectInspector oi1 = (StructObjectInspector) ObjectInspectorFactory + .getReflectionObjectInspector(TextStringMapHolder.class, ObjectInspectorOptions.JAVA); + + LazySimpleSerDe serde = new LazySimpleSerDe(); + Configuration conf = new Configuration(); + Properties tbl = new Properties(); + tbl.setProperty(Constants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1)); + tbl.setProperty(Constants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1)); + SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl, + LazySimpleSerDe.class.getName()); + serde.initialize(conf, tbl); + ObjectInspector oi2 = serde.getObjectInspector(); + + Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams); + + int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new CrossMapEqualComparer()); + assertEquals(0, rc); + + // equal maps + o1.mMap.put(new Text("42"), "The answer to Life, Universe And Everything"); + o1.mMap.put(new Text("1729"), "A taxi cab number"); + o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams); + rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new CrossMapEqualComparer()); + assertEquals(0, rc); + + // unequal maps + o1.mMap.put(new Text("1729"), "Hardy-Ramanujan Number"); + rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new CrossMapEqualComparer()); + assertFalse(0 == rc); + } + + public static class StringTextMapHolder { + Map mMap; + + public StringTextMapHolder() { + mMap = new TreeMap(); + } + } + + Object serializeAndDeserialize(StringTextMapHolder o1, StructObjectInspector oi1, + LazySimpleSerDe serde, + SerDeParameters serdeParams) throws IOException, SerDeException { + ByteStream.Output serializeStream = new ByteStream.Output(); + LazySimpleSerDe.serialize(serializeStream, o1, oi1, serdeParams + .getSeparators(), 0, serdeParams.getNullSequence(), serdeParams + .isEscaped(), serdeParams.getEscapeChar(), serdeParams + .getNeedsEscape()); + Text t = new Text(serializeStream.toByteArray()); + return serde.deserialize(t); + } + + public void testIncompatibleType() throws SerDeException, IOException { + // empty maps + StringTextMapHolder o1 = new StringTextMapHolder(); + StructObjectInspector oi1 = (StructObjectInspector) ObjectInspectorFactory + .getReflectionObjectInspector(StringTextMapHolder.class, ObjectInspectorOptions.JAVA); + + LazySimpleSerDe serde = new LazySimpleSerDe(); + Configuration conf = new Configuration(); + Properties tbl = new Properties(); + tbl.setProperty(Constants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1)); + tbl.setProperty(Constants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1)); + SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl, + LazySimpleSerDe.class.getName()); + serde.initialize(conf, tbl); + ObjectInspector oi2 = serde.getObjectInspector(); + + Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams); + + int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new CrossMapEqualComparer()); + assertEquals(0, rc); + + // equal maps + o1.mMap.put("42", new Text("The answer to Life, Universe And Everything")); + o1.mMap.put("1729", new Text("A taxi cab number")); + o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams); + rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new CrossMapEqualComparer()); + assertEquals(0, rc); + + // unequal maps + o1.mMap.put("1729", new Text("Hardy-Ramanujan Number")); + rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new CrossMapEqualComparer()); + assertFalse(0 == rc); + + } + +} diff --git serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestFullMapEqualComparer.java serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestFullMapEqualComparer.java new file mode 100644 index 0000000..4e5c7ca --- /dev/null +++ serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestFullMapEqualComparer.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector; + +import java.util.Map; +import java.util.TreeMap; + +import junit.framework.TestCase; + +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions; + +public class TestFullMapEqualComparer extends TestCase { + + public static class IntegerIntegerMapHolder { + Map mMap; + + public IntegerIntegerMapHolder() { + mMap = new TreeMap(); + } + } + + public void testAntiSymmetry() { + IntegerIntegerMapHolder o1 = new IntegerIntegerMapHolder(); + IntegerIntegerMapHolder o2 = new IntegerIntegerMapHolder(); + + ObjectInspector oi = ObjectInspectorFactory + .getReflectionObjectInspector(IntegerIntegerMapHolder.class, ObjectInspectorOptions.JAVA); + + o1.mMap.put(1, 1); + + o2.mMap.put(0, 99); + + { // not anti-symmetric + int rc12 = ObjectInspectorUtils.compare(o1, oi, o2, oi, new SimpleMapEqualComparer()); + assertTrue(rc12 > 0); + int rc21 = ObjectInspectorUtils.compare(o2, oi, o1, oi, new SimpleMapEqualComparer()); + assertTrue(rc21 > 0); + } + { // not anti-symmetric + int rc12 = ObjectInspectorUtils.compare(o1, oi, o2, oi, new CrossMapEqualComparer()); + assertTrue(rc12 > 0); + int rc21 = ObjectInspectorUtils.compare(o2, oi, o1, oi, new CrossMapEqualComparer()); + assertTrue(rc21 > 0); + } + {// anti-symmetric + int rc12 = ObjectInspectorUtils.compare(o1, oi, o2, oi, new FullMapEqualComparer()); + assertTrue(rc12 > 0); + int rc21 = ObjectInspectorUtils.compare(o2, oi, o1, oi, new FullMapEqualComparer()); + assertTrue(rc21 < 0); + } + + } + + public void testTransitivity() { + IntegerIntegerMapHolder o1 = new IntegerIntegerMapHolder(); + IntegerIntegerMapHolder o2 = new IntegerIntegerMapHolder(); + IntegerIntegerMapHolder o3 = new IntegerIntegerMapHolder(); + + ObjectInspector oi = ObjectInspectorFactory + .getReflectionObjectInspector(IntegerIntegerMapHolder.class, ObjectInspectorOptions.JAVA); + + o1.mMap.put(1, 1); + o1.mMap.put(99, 99); + + o2.mMap.put(0, 99); + o2.mMap.put(99, 99); + + o3.mMap.put(0, 1); + o3.mMap.put(1, 99); + + { // non-transitive + int rc12 = ObjectInspectorUtils.compare(o1, oi, o2, oi, new SimpleMapEqualComparer()); + assertTrue(rc12 > 0); + int rc23 = ObjectInspectorUtils.compare(o2, oi, o3, oi, new SimpleMapEqualComparer()); + assertTrue(rc23 > 0); + int rc13 = ObjectInspectorUtils.compare(o1, oi, o3, oi, new SimpleMapEqualComparer()); + assertTrue(rc13 < 0); + } + { // non-transitive + int rc12 = ObjectInspectorUtils.compare(o1, oi, o2, oi, new CrossMapEqualComparer()); + assertTrue(rc12 > 0); + int rc23 = ObjectInspectorUtils.compare(o2, oi, o3, oi, new CrossMapEqualComparer()); + assertTrue(rc23 > 0); + int rc13 = ObjectInspectorUtils.compare(o1, oi, o3, oi, new CrossMapEqualComparer()); + assertTrue(rc13 < 0); + } + {// transitive + int rc12 = ObjectInspectorUtils.compare(o1, oi, o2, oi, new FullMapEqualComparer()); + assertTrue(rc12 > 0); + int rc23 = ObjectInspectorUtils.compare(o2, oi, o3, oi, new FullMapEqualComparer()); + assertTrue(rc23 > 0); + int rc13 = ObjectInspectorUtils.compare(o1, oi, o3, oi, new FullMapEqualComparer()); + assertTrue(rc13 > 0); + } + } + +} diff --git serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestSimpleMapEqualComparer.java serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestSimpleMapEqualComparer.java new file mode 100644 index 0000000..982f096 --- /dev/null +++ serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestSimpleMapEqualComparer.java @@ -0,0 +1,173 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector; + +import java.io.IOException; +import java.util.Map; +import java.util.Properties; +import java.util.TreeMap; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde.Constants; +import org.apache.hadoop.hive.serde2.ByteStream; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; +import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions; +import org.apache.hadoop.io.Text; + +import junit.framework.TestCase; + +public class TestSimpleMapEqualComparer extends TestCase { + + public static class IntegerStringMapHolder { + Map mMap; + + public IntegerStringMapHolder() { + mMap = new TreeMap(); + } + } + + public void testSameType() { + // empty maps + IntegerStringMapHolder o1 = new IntegerStringMapHolder(); + IntegerStringMapHolder o2 = new IntegerStringMapHolder(); + ObjectInspector oi1 = ObjectInspectorFactory + .getReflectionObjectInspector(IntegerStringMapHolder.class, ObjectInspectorOptions.JAVA); + int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi1, new SimpleMapEqualComparer()); + assertEquals(0, rc); + + // equal maps + o1.mMap.put(42, "The answer to Life, Universe And Everything"); + o2.mMap.put(42, "The answer to Life, Universe And Everything"); + + o1.mMap.put(1729, "A taxi cab number"); + o2.mMap.put(1729, "A taxi cab number"); + rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi1, new SimpleMapEqualComparer()); + assertEquals(0, rc); + + // unequal maps + o2.mMap.put(1729, "Hardy-Ramanujan Number"); + rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi1, new SimpleMapEqualComparer()); + assertFalse(0 == rc); + } + + public static class TextStringMapHolder { + Map mMap; + + public TextStringMapHolder() { + mMap = new TreeMap(); + } + } + + Object serializeAndDeserialize(TextStringMapHolder o1, StructObjectInspector oi1, + LazySimpleSerDe serde, + SerDeParameters serdeParams) throws IOException, SerDeException { + ByteStream.Output serializeStream = new ByteStream.Output(); + LazySimpleSerDe.serialize(serializeStream, o1, oi1, serdeParams + .getSeparators(), 0, serdeParams.getNullSequence(), serdeParams + .isEscaped(), serdeParams.getEscapeChar(), serdeParams + .getNeedsEscape()); + Text t = new Text(serializeStream.toByteArray()); + return serde.deserialize(t); + } + + public void testCompatibleType() throws SerDeException, IOException { + // empty maps + TextStringMapHolder o1 = new TextStringMapHolder(); + StructObjectInspector oi1 = (StructObjectInspector) ObjectInspectorFactory + .getReflectionObjectInspector(TextStringMapHolder.class, ObjectInspectorOptions.JAVA); + + LazySimpleSerDe serde = new LazySimpleSerDe(); + Configuration conf = new Configuration(); + Properties tbl = new Properties(); + tbl.setProperty(Constants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1)); + tbl.setProperty(Constants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1)); + SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl, + LazySimpleSerDe.class.getName()); + serde.initialize(conf, tbl); + ObjectInspector oi2 = serde.getObjectInspector(); + + Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams); + + int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer()); + assertEquals(0, rc); + + // equal maps + o1.mMap.put(new Text("42"), "The answer to Life, Universe And Everything"); + o1.mMap.put(new Text("1729"), "A taxi cab number"); + o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams); + rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer()); + assertEquals(0, rc); + + // unequal maps + o1.mMap.put(new Text("1729"), "Hardy-Ramanujan Number"); + rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer()); + assertFalse(0 == rc); + } + + public static class StringTextMapHolder { + Map mMap; + + public StringTextMapHolder() { + mMap = new TreeMap(); + } + } + + Object serializeAndDeserialize(StringTextMapHolder o1, StructObjectInspector oi1, + LazySimpleSerDe serde, + SerDeParameters serdeParams) throws IOException, SerDeException { + ByteStream.Output serializeStream = new ByteStream.Output(); + LazySimpleSerDe.serialize(serializeStream, o1, oi1, serdeParams + .getSeparators(), 0, serdeParams.getNullSequence(), serdeParams + .isEscaped(), serdeParams.getEscapeChar(), serdeParams + .getNeedsEscape()); + Text t = new Text(serializeStream.toByteArray()); + return serde.deserialize(t); + } + + public void testIncompatibleType() throws SerDeException, IOException { + // empty maps + StringTextMapHolder o1 = new StringTextMapHolder(); + StructObjectInspector oi1 = (StructObjectInspector) ObjectInspectorFactory + .getReflectionObjectInspector(StringTextMapHolder.class, ObjectInspectorOptions.JAVA); + + LazySimpleSerDe serde = new LazySimpleSerDe(); + Configuration conf = new Configuration(); + Properties tbl = new Properties(); + tbl.setProperty(Constants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1)); + tbl.setProperty(Constants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1)); + SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl, + LazySimpleSerDe.class.getName()); + serde.initialize(conf, tbl); + ObjectInspector oi2 = serde.getObjectInspector(); + + Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams); + + int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer()); + assertEquals(0, rc); + + // equal maps + o1.mMap.put("42", new Text("The answer to Life, Universe And Everything")); + o1.mMap.put("1729", new Text("A taxi cab number")); + o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams); + rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer()); + assertFalse(0 == rc); + } + +}