commit 654bf64918a4738b78cbc9a3f7f5ca2229bd1eac Author: Owen O'Malley Date: Thu Jul 14 16:53:41 2016 -0700 HIVE-14242. Backport of ORC-53. diff --git orc/src/java/org/apache/orc/TypeDescription.java orc/src/java/org/apache/orc/TypeDescription.java index ffe3c1f..1585e43 100644 --- orc/src/java/org/apache/orc/TypeDescription.java +++ orc/src/java/org/apache/orc/TypeDescription.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -37,12 +38,61 @@ /** * This is the description of the types in an ORC file. */ -public class TypeDescription { +public class TypeDescription + implements Comparable, Serializable { private static final int MAX_PRECISION = 38; private static final int MAX_SCALE = 38; private static final int DEFAULT_PRECISION = 38; private static final int DEFAULT_SCALE = 10; private static final int DEFAULT_LENGTH = 256; + + @Override + public int compareTo(TypeDescription other) { + if (this == other) { + return 0; + } else if (other == null) { + return -1; + } else { + int result = category.compareTo(other.category); + if (result == 0) { + switch (category) { + case CHAR: + case VARCHAR: + return maxLength - other.maxLength; + case DECIMAL: + if (precision != other.precision) { + return precision - other.precision; + } + return scale - other.scale; + case UNION: + case LIST: + case MAP: + if (children.size() != other.children.size()) { + return children.size() - other.children.size(); + } + for(int c=0; result == 0 && c < children.size(); ++c) { + result = children.get(c).compareTo(other.children.get(c)); + } + break; + case STRUCT: + if (children.size() != other.children.size()) { + return children.size() - other.children.size(); + } + for(int c=0; result == 0 && c < children.size(); ++c) { + result = fieldNames.get(c).compareTo(other.fieldNames.get(c)); + if (result == 0) { + result = children.get(c).compareTo(other.children.get(c)); + } + } + break; + default: + // PASS + } + } + return result; + } + } + public enum Category { BOOLEAN("boolean", true), BYTE("tinyint", true), @@ -278,12 +328,18 @@ public TypeDescription clone() { @Override public int hashCode() { - return getId(); + long result = category.ordinal() * 4241 + maxLength + precision * 13 + scale; + if (children != null) { + for(TypeDescription child: children) { + result = result * 6959 + child.hashCode(); + } + } + return (int) result; } @Override public boolean equals(Object other) { - if (other == null || other.getClass() != TypeDescription.class) { + if (other == null || !(other instanceof TypeDescription)) { return false; } if (other == this) { @@ -291,8 +347,6 @@ public boolean equals(Object other) { } TypeDescription castOther = (TypeDescription) other; if (category != castOther.category || - getId() != castOther.getId() || - getMaximumId() != castOther.getMaximumId() || maxLength != castOther.maxLength || scale != castOther.scale || precision != castOther.precision) { diff --git orc/src/java/org/apache/orc/impl/SchemaEvolution.java orc/src/java/org/apache/orc/impl/SchemaEvolution.java index a6c1d60..61a3f85 100644 --- orc/src/java/org/apache/orc/impl/SchemaEvolution.java +++ orc/src/java/org/apache/orc/impl/SchemaEvolution.java @@ -33,7 +33,7 @@ * has been schema evolution. */ public class SchemaEvolution { - private final Map readerToFile; + private final Map readerToFile; private final boolean[] included; private final TypeDescription readerSchema; private static final Log LOG = LogFactory.getLog(SchemaEvolution.class); @@ -70,7 +70,7 @@ public TypeDescription getFileType(TypeDescription readerType) { result = null; } } else { - result = readerToFile.get(readerType); + result = readerToFile.get(readerType.getId()); } return result; } @@ -141,7 +141,7 @@ void buildMapping(TypeDescription fileType, isOk = ConvertTreeReaderFactory.canConvert(fileType, readerType); } if (isOk) { - readerToFile.put(readerType, fileType); + readerToFile.put(readerType.getId(), fileType); } else { throw new IOException( String.format( diff --git orc/src/test/org/apache/orc/TestTypeDescription.java orc/src/test/org/apache/orc/TestTypeDescription.java index 0ac1e64..27516be 100644 --- orc/src/test/org/apache/orc/TestTypeDescription.java +++ orc/src/test/org/apache/orc/TestTypeDescription.java @@ -65,4 +65,27 @@ public void testJson() { " \"f6\": {\"category\": \"char\", \"id\": 8, \"max\": 8, \"length\": 100}]}", struct.toJson()); } + + @Test + public void testEquals() { + TypeDescription type1 = + TypeDescription.createStruct() + .addField("a", TypeDescription.createInt()) + .addField("b", TypeDescription.createStruct() + .addField("x", TypeDescription.createString()) + .addField("y", TypeDescription.createBinary()) + .addField("z", TypeDescription.createDouble())) + .addField("c", TypeDescription.createString()); + assertEquals(0, type1.getId()); + assertEquals(6, type1.getMaximumId()); + TypeDescription type2 = + TypeDescription.createStruct() + .addField("x", TypeDescription.createString()) + .addField("y", TypeDescription.createBinary()) + .addField("z", TypeDescription.createDouble()); + assertEquals(0, type2.getId()); + assertEquals(3, type2.getMaximumId()); + assertEquals(type2, type1.getChildren().get(1)); + assertEquals(type2.hashCode(), type1.getChildren().get(1).hashCode()); + } }