diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java index f7843c1..21fa2f7 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java @@ -1,116 +1,116 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.objectinspector; - -import java.util.List; - -/** - * StandardUnionObjectInspector works on union data that is stored as - * UnionObject. - * It holds the list of the object inspectors corresponding to each type of the - * object the Union can hold. The UniobObject has tag followed by the object - * it is holding. - * - * Always use the {@link ObjectInspectorFactory} to create new ObjectInspector - * objects, instead of directly creating an instance of this class. - */ -public class StandardUnionObjectInspector implements UnionObjectInspector { - List ois; - - public StandardUnionObjectInspector(List ois) { - this.ois = ois; - } - - public List getObjectInspectors() { - return ois; - } - - public static class StandardUnion implements UnionObject { - protected byte tag; - protected Object object; - - public StandardUnion() { - } - - public StandardUnion(byte tag, Object object) { - this.tag = tag; - this.object = object; - } - - public void setObject(Object o) { - this.object = o; - } - - public void setTag(byte tag) { - this.tag = tag; - } - - @Override - public Object getObject() { - return object; - } - - @Override - public byte getTag() { - return tag; - } - - @Override - public String toString() { - return tag + ":" + object; - } - } - - /** - * Return the tag of the object. - */ - public byte getTag(Object o) { - if (o == null) { - return -1; - } - return ((UnionObject) o).getTag(); - } - - /** - * Return the field based on the tag value associated with the Object. - */ - public Object getField(Object o) { - if (o == null) { - return null; - } - return ((UnionObject) o).getObject(); - } - - public Category getCategory() { - return Category.UNION; - } - - public String getTypeName() { - return ObjectInspectorUtils.getStandardUnionTypeName(this); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(getClass().getName()); - sb.append(getTypeName()); - return sb.toString(); - } - -} +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.objectinspector; + +import java.util.List; + +/** + * StandardUnionObjectInspector works on union data that is stored as + * UnionObject. + * It holds the list of the object inspectors corresponding to each type of the + * object the Union can hold. The UniobObject has tag followed by the object + * it is holding. + * + * Always use the {@link ObjectInspectorFactory} to create new ObjectInspector + * objects, instead of directly creating an instance of this class. + */ +public class StandardUnionObjectInspector implements UnionObjectInspector { + List ois; + + public StandardUnionObjectInspector(List ois) { + this.ois = ois; + } + + public List getObjectInspectors() { + return ois; + } + + public static class StandardUnion implements UnionObject { + protected byte tag; + protected Object object; + + public StandardUnion() { + } + + public StandardUnion(byte tag, Object object) { + this.tag = tag; + this.object = object; + } + + public void setObject(Object o) { + this.object = o; + } + + public void setTag(byte tag) { + this.tag = tag; + } + + @Override + public Object getObject() { + return object; + } + + @Override + public byte getTag() { + return tag; + } + + @Override + public String toString() { + return tag + ":" + object; + } + } + + /** + * Return the tag of the object. + */ + public byte getTag(Object o) { + if (o == null) { + return -1; + } + return ((UnionObject) o).getTag(); + } + + /** + * Return the field based on the tag value associated with the Object. + */ + public Object getField(Object o) { + if (o == null) { + return null; + } + return ((UnionObject) o).getObject(); + } + + public Category getCategory() { + return Category.UNION; + } + + public String getTypeName() { + return ObjectInspectorUtils.getStandardUnionTypeName(this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getName()); + sb.append(getTypeName()); + return sb.toString(); + } + +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionObjectInspector.java index 8dec1af..26de76a 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionObjectInspector.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionObjectInspector.java @@ -1,49 +1,49 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.objectinspector; - -import java.util.List; - -/** - * UnionObjectInspector works on union data that is stored as UnionObject. - * - * It holds the list of the object inspectors corresponding to each type of the - * object the Union can hold. - * - * UnionObjectInspector. - * - */ -public interface UnionObjectInspector extends ObjectInspector { - - /** - * Returns the array of ObjectInspectors that are for each of the tags. - */ - List getObjectInspectors(); - - /** - * Return the tag of the object. - */ - byte getTag(Object o); - - /** - * Return the field based on the tag associated with the Object. - */ - Object getField(Object o); - -} +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.objectinspector; + +import java.util.List; + +/** + * UnionObjectInspector works on union data that is stored as UnionObject. + * + * It holds the list of the object inspectors corresponding to each type of the + * object the Union can hold. + * + * UnionObjectInspector. + * + */ +public interface UnionObjectInspector extends ObjectInspector { + + /** + * Returns the array of ObjectInspectors that are for each of the tags. + */ + List getObjectInspectors(); + + /** + * Return the tag of the object. + */ + byte getTag(Object o); + + /** + * Return the field based on the tag associated with the Object. + */ + Object getField(Object o); + +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/ListTypeInfo.java serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/ListTypeInfo.java index fe406cf..48fb653 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/ListTypeInfo.java +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/ListTypeInfo.java @@ -1,89 +1,89 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.typeinfo; - -import java.io.Serializable; - -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; - -/** - * A List Type has homogeneous elements. All elements of the List has the same - * TypeInfo which is returned by getListElementTypeInfo. - * - * Always use the TypeInfoFactory to create new TypeInfo objects, instead of - * directly creating an instance of this class. - */ -public final class ListTypeInfo extends TypeInfo implements Serializable { - - private static final long serialVersionUID = 1L; - private TypeInfo listElementTypeInfo; - - /** - * For java serialization use only. - */ - public ListTypeInfo() { - } - - @Override - public String getTypeName() { - return org.apache.hadoop.hive.serde.serdeConstants.LIST_TYPE_NAME + "<" - + listElementTypeInfo.getTypeName() + ">"; - } - - /** - * For java serialization use only. - */ - public void setListElementTypeInfo(TypeInfo listElementTypeInfo) { - this.listElementTypeInfo = listElementTypeInfo; - } - - /** - * For TypeInfoFactory use only. - */ - ListTypeInfo(TypeInfo elementTypeInfo) { - listElementTypeInfo = elementTypeInfo; - } - - @Override - public Category getCategory() { - return Category.LIST; - } - - public TypeInfo getListElementTypeInfo() { - return listElementTypeInfo; - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (!(other instanceof ListTypeInfo)) { - return false; - } - return getListElementTypeInfo().equals( - ((ListTypeInfo) other).getListElementTypeInfo()); - } - - @Override - public int hashCode() { - return listElementTypeInfo.hashCode(); - } - -} +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.typeinfo; + +import java.io.Serializable; + +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; + +/** + * A List Type has homogeneous elements. All elements of the List has the same + * TypeInfo which is returned by getListElementTypeInfo. + * + * Always use the TypeInfoFactory to create new TypeInfo objects, instead of + * directly creating an instance of this class. + */ +public final class ListTypeInfo extends TypeInfo implements Serializable { + + private static final long serialVersionUID = 1L; + private TypeInfo listElementTypeInfo; + + /** + * For java serialization use only. + */ + public ListTypeInfo() { + } + + @Override + public String getTypeName() { + return org.apache.hadoop.hive.serde.serdeConstants.LIST_TYPE_NAME + "<" + + listElementTypeInfo.getTypeName() + ">"; + } + + /** + * For java serialization use only. + */ + public void setListElementTypeInfo(TypeInfo listElementTypeInfo) { + this.listElementTypeInfo = listElementTypeInfo; + } + + /** + * For TypeInfoFactory use only. + */ + ListTypeInfo(TypeInfo elementTypeInfo) { + listElementTypeInfo = elementTypeInfo; + } + + @Override + public Category getCategory() { + return Category.LIST; + } + + public TypeInfo getListElementTypeInfo() { + return listElementTypeInfo; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof ListTypeInfo)) { + return false; + } + return getListElementTypeInfo().equals( + ((ListTypeInfo) other).getListElementTypeInfo()); + } + + @Override + public int hashCode() { + return listElementTypeInfo.hashCode(); + } + +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/MapTypeInfo.java serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/MapTypeInfo.java index cc499ae..001d7f4 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/MapTypeInfo.java +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/MapTypeInfo.java @@ -1,105 +1,105 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.typeinfo; - -import java.io.Serializable; - -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; - -/** - * A Map Type has homogeneous keys and homogeneous values. All keys of the Map - * have the same TypeInfo, which is returned by getMapKeyTypeInfo(); and all - * values of the Map has the same TypeInfo, which is returned by - * getMapValueTypeInfo(). - * - * Always use the TypeInfoFactory to create new TypeInfo objects, instead of - * directly creating an instance of this class. - */ -public final class MapTypeInfo extends TypeInfo implements Serializable { - - private static final long serialVersionUID = 1L; - - private TypeInfo mapKeyTypeInfo; - private TypeInfo mapValueTypeInfo; - - /** - * For java serialization use only. - */ - public MapTypeInfo() { - } - - @Override - public String getTypeName() { - return org.apache.hadoop.hive.serde.serdeConstants.MAP_TYPE_NAME + "<" - + mapKeyTypeInfo.getTypeName() + "," + mapValueTypeInfo.getTypeName() - + ">"; - } - - /** - * For java serialization use only. - */ - public void setMapKeyTypeInfo(TypeInfo mapKeyTypeInfo) { - this.mapKeyTypeInfo = mapKeyTypeInfo; - } - - /** - * For java serialization use only. - */ - public void setMapValueTypeInfo(TypeInfo mapValueTypeInfo) { - this.mapValueTypeInfo = mapValueTypeInfo; - } - - // For TypeInfoFactory use only - MapTypeInfo(TypeInfo keyTypeInfo, TypeInfo valueTypeInfo) { - mapKeyTypeInfo = keyTypeInfo; - mapValueTypeInfo = valueTypeInfo; - } - - @Override - public Category getCategory() { - return Category.MAP; - } - - public TypeInfo getMapKeyTypeInfo() { - return mapKeyTypeInfo; - } - - public TypeInfo getMapValueTypeInfo() { - return mapValueTypeInfo; - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (!(other instanceof MapTypeInfo)) { - return false; - } - MapTypeInfo o = (MapTypeInfo) other; - return o.getMapKeyTypeInfo().equals(getMapKeyTypeInfo()) - && o.getMapValueTypeInfo().equals(getMapValueTypeInfo()); - } - - @Override - public int hashCode() { - return mapKeyTypeInfo.hashCode() ^ mapValueTypeInfo.hashCode(); - } - -} +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.typeinfo; + +import java.io.Serializable; + +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; + +/** + * A Map Type has homogeneous keys and homogeneous values. All keys of the Map + * have the same TypeInfo, which is returned by getMapKeyTypeInfo(); and all + * values of the Map has the same TypeInfo, which is returned by + * getMapValueTypeInfo(). + * + * Always use the TypeInfoFactory to create new TypeInfo objects, instead of + * directly creating an instance of this class. + */ +public final class MapTypeInfo extends TypeInfo implements Serializable { + + private static final long serialVersionUID = 1L; + + private TypeInfo mapKeyTypeInfo; + private TypeInfo mapValueTypeInfo; + + /** + * For java serialization use only. + */ + public MapTypeInfo() { + } + + @Override + public String getTypeName() { + return org.apache.hadoop.hive.serde.serdeConstants.MAP_TYPE_NAME + "<" + + mapKeyTypeInfo.getTypeName() + "," + mapValueTypeInfo.getTypeName() + + ">"; + } + + /** + * For java serialization use only. + */ + public void setMapKeyTypeInfo(TypeInfo mapKeyTypeInfo) { + this.mapKeyTypeInfo = mapKeyTypeInfo; + } + + /** + * For java serialization use only. + */ + public void setMapValueTypeInfo(TypeInfo mapValueTypeInfo) { + this.mapValueTypeInfo = mapValueTypeInfo; + } + + // For TypeInfoFactory use only + MapTypeInfo(TypeInfo keyTypeInfo, TypeInfo valueTypeInfo) { + mapKeyTypeInfo = keyTypeInfo; + mapValueTypeInfo = valueTypeInfo; + } + + @Override + public Category getCategory() { + return Category.MAP; + } + + public TypeInfo getMapKeyTypeInfo() { + return mapKeyTypeInfo; + } + + public TypeInfo getMapValueTypeInfo() { + return mapValueTypeInfo; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof MapTypeInfo)) { + return false; + } + MapTypeInfo o = (MapTypeInfo) other; + return o.getMapKeyTypeInfo().equals(getMapKeyTypeInfo()) + && o.getMapValueTypeInfo().equals(getMapValueTypeInfo()); + } + + @Override + public int hashCode() { + return mapKeyTypeInfo.hashCode() ^ mapValueTypeInfo.hashCode(); + } + +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java index c81a037..46d3f3d 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java @@ -1,100 +1,100 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.typeinfo; - -import java.io.Serializable; - -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; - -/** - * There are limited number of Primitive Types. All Primitive Types are defined - * by TypeInfoFactory.isPrimitiveClass(). - * - * Always use the TypeInfoFactory to create new TypeInfo objects, instead of - * directly creating an instance of this class. - */ -public final class PrimitiveTypeInfo extends TypeInfo implements Serializable { - - private static final long serialVersionUID = 1L; - - private String typeName; - - /** - * For java serialization use only. - */ - public PrimitiveTypeInfo() { - } - - /** - * For TypeInfoFactory use only. - */ - PrimitiveTypeInfo(String typeName) { - this.typeName = typeName; - } - - /** - * Returns the category of this TypeInfo. - */ - @Override - public Category getCategory() { - return Category.PRIMITIVE; - } - - public PrimitiveCategory getPrimitiveCategory() { - return PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(typeName).primitiveCategory; - } - - public Class getPrimitiveWritableClass() { - return PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(typeName).primitiveWritableClass; - } - - public Class getPrimitiveJavaClass() { - return PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(typeName).primitiveJavaClass; - } - - // The following 2 methods are for java serialization use only. - public void setTypeName(String typeName) { - this.typeName = typeName; - } - - @Override - public String getTypeName() { - return typeName; - } - - /** - * Compare if 2 TypeInfos are the same. We use TypeInfoFactory to cache - * TypeInfos, so we only need to compare the Object pointer. - */ - @Override - public boolean equals(Object other) { - return this == other; - } - - /** - * Generate the hashCode for this TypeInfo. - */ - @Override - public int hashCode() { - return typeName.hashCode(); - } - -} +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.typeinfo; + +import java.io.Serializable; + +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; + +/** + * There are limited number of Primitive Types. All Primitive Types are defined + * by TypeInfoFactory.isPrimitiveClass(). + * + * Always use the TypeInfoFactory to create new TypeInfo objects, instead of + * directly creating an instance of this class. + */ +public final class PrimitiveTypeInfo extends TypeInfo implements Serializable { + + private static final long serialVersionUID = 1L; + + private String typeName; + + /** + * For java serialization use only. + */ + public PrimitiveTypeInfo() { + } + + /** + * For TypeInfoFactory use only. + */ + PrimitiveTypeInfo(String typeName) { + this.typeName = typeName; + } + + /** + * Returns the category of this TypeInfo. + */ + @Override + public Category getCategory() { + return Category.PRIMITIVE; + } + + public PrimitiveCategory getPrimitiveCategory() { + return PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(typeName).primitiveCategory; + } + + public Class getPrimitiveWritableClass() { + return PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(typeName).primitiveWritableClass; + } + + public Class getPrimitiveJavaClass() { + return PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(typeName).primitiveJavaClass; + } + + // The following 2 methods are for java serialization use only. + public void setTypeName(String typeName) { + this.typeName = typeName; + } + + @Override + public String getTypeName() { + return typeName; + } + + /** + * Compare if 2 TypeInfos are the same. We use TypeInfoFactory to cache + * TypeInfos, so we only need to compare the Object pointer. + */ + @Override + public boolean equals(Object other) { + return this == other; + } + + /** + * Generate the hashCode for this TypeInfo. + */ + @Override + public int hashCode() { + return typeName.hashCode(); + } + +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java index 0ac179d..465bf31 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java @@ -1,147 +1,147 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.typeinfo; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; - -/** - * StructTypeInfo represents the TypeInfo of a struct. A struct contains one or - * more fields each of which has a unique name and its own TypeInfo. Different - * fields can have the same or different TypeInfo. - * - * Always use the TypeInfoFactory to create new TypeInfo objects, instead of - * directly creating an instance of this class. - */ -public final class StructTypeInfo extends TypeInfo implements Serializable { - - private static final long serialVersionUID = 1L; - - private ArrayList allStructFieldNames; - private ArrayList allStructFieldTypeInfos; - - /** - * For java serialization use only. - */ - public StructTypeInfo() { - } - - @Override - public String getTypeName() { - StringBuilder sb = new StringBuilder(); - sb.append(serdeConstants.STRUCT_TYPE_NAME + "<"); - for (int i = 0; i < allStructFieldNames.size(); i++) { - if (i > 0) { - sb.append(","); - } - sb.append(allStructFieldNames.get(i)); - sb.append(":"); - sb.append(allStructFieldTypeInfos.get(i).getTypeName()); - } - sb.append(">"); - return sb.toString(); - } - - /** - * For java serialization use only. - */ - public void setAllStructFieldNames(ArrayList allStructFieldNames) { - this.allStructFieldNames = allStructFieldNames; - } - - /** - * For java serialization use only. - */ - public void setAllStructFieldTypeInfos( - ArrayList allStructFieldTypeInfos) { - this.allStructFieldTypeInfos = allStructFieldTypeInfos; - } - - /** - * For TypeInfoFactory use only. - */ - StructTypeInfo(List names, List typeInfos) { - allStructFieldNames = new ArrayList(names); - allStructFieldTypeInfos = new ArrayList(typeInfos); - } - - @Override - public Category getCategory() { - return Category.STRUCT; - } - - public ArrayList getAllStructFieldNames() { - return allStructFieldNames; - } - - public ArrayList getAllStructFieldTypeInfos() { - return allStructFieldTypeInfos; - } - - public TypeInfo getStructFieldTypeInfo(String field) { - String fieldLowerCase = field.toLowerCase(); - for (int i = 0; i < allStructFieldNames.size(); i++) { - if (fieldLowerCase.equals(allStructFieldNames.get(i))) { - return allStructFieldTypeInfos.get(i); - } - } - throw new RuntimeException("cannot find field " + field - + "(lowercase form: " + fieldLowerCase + ") in " + allStructFieldNames); - // return null; - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (!(other instanceof StructTypeInfo)) { - return false; - } - StructTypeInfo o = (StructTypeInfo) other; - Iterator namesIterator = getAllStructFieldNames().iterator(); - Iterator otherNamesIterator = o.getAllStructFieldNames().iterator(); - - // Compare the field names using ignore-case semantics - while (namesIterator.hasNext() && otherNamesIterator.hasNext()) { - if (!namesIterator.next().equalsIgnoreCase(otherNamesIterator.next())) { - return false; - } - } - - // Different number of field names - if (namesIterator.hasNext() || otherNamesIterator.hasNext()) { - return false; - } - - // Compare the field types - return o.getAllStructFieldTypeInfos().equals(getAllStructFieldTypeInfos()); - } - - @Override - public int hashCode() { - return allStructFieldNames.hashCode() ^ allStructFieldTypeInfos.hashCode(); - } - -} +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.typeinfo; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; + +/** + * StructTypeInfo represents the TypeInfo of a struct. A struct contains one or + * more fields each of which has a unique name and its own TypeInfo. Different + * fields can have the same or different TypeInfo. + * + * Always use the TypeInfoFactory to create new TypeInfo objects, instead of + * directly creating an instance of this class. + */ +public final class StructTypeInfo extends TypeInfo implements Serializable { + + private static final long serialVersionUID = 1L; + + private ArrayList allStructFieldNames; + private ArrayList allStructFieldTypeInfos; + + /** + * For java serialization use only. + */ + public StructTypeInfo() { + } + + @Override + public String getTypeName() { + StringBuilder sb = new StringBuilder(); + sb.append(serdeConstants.STRUCT_TYPE_NAME + "<"); + for (int i = 0; i < allStructFieldNames.size(); i++) { + if (i > 0) { + sb.append(","); + } + sb.append(allStructFieldNames.get(i)); + sb.append(":"); + sb.append(allStructFieldTypeInfos.get(i).getTypeName()); + } + sb.append(">"); + return sb.toString(); + } + + /** + * For java serialization use only. + */ + public void setAllStructFieldNames(ArrayList allStructFieldNames) { + this.allStructFieldNames = allStructFieldNames; + } + + /** + * For java serialization use only. + */ + public void setAllStructFieldTypeInfos( + ArrayList allStructFieldTypeInfos) { + this.allStructFieldTypeInfos = allStructFieldTypeInfos; + } + + /** + * For TypeInfoFactory use only. + */ + StructTypeInfo(List names, List typeInfos) { + allStructFieldNames = new ArrayList(names); + allStructFieldTypeInfos = new ArrayList(typeInfos); + } + + @Override + public Category getCategory() { + return Category.STRUCT; + } + + public ArrayList getAllStructFieldNames() { + return allStructFieldNames; + } + + public ArrayList getAllStructFieldTypeInfos() { + return allStructFieldTypeInfos; + } + + public TypeInfo getStructFieldTypeInfo(String field) { + String fieldLowerCase = field.toLowerCase(); + for (int i = 0; i < allStructFieldNames.size(); i++) { + if (fieldLowerCase.equals(allStructFieldNames.get(i))) { + return allStructFieldTypeInfos.get(i); + } + } + throw new RuntimeException("cannot find field " + field + + "(lowercase form: " + fieldLowerCase + ") in " + allStructFieldNames); + // return null; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof StructTypeInfo)) { + return false; + } + StructTypeInfo o = (StructTypeInfo) other; + Iterator namesIterator = getAllStructFieldNames().iterator(); + Iterator otherNamesIterator = o.getAllStructFieldNames().iterator(); + + // Compare the field names using ignore-case semantics + while (namesIterator.hasNext() && otherNamesIterator.hasNext()) { + if (!namesIterator.next().equalsIgnoreCase(otherNamesIterator.next())) { + return false; + } + } + + // Different number of field names + if (namesIterator.hasNext() || otherNamesIterator.hasNext()) { + return false; + } + + // Compare the field types + return o.getAllStructFieldTypeInfos().equals(getAllStructFieldTypeInfos()); + } + + @Override + public int hashCode() { + return allStructFieldNames.hashCode() ^ allStructFieldTypeInfos.hashCode(); + } + +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java index 2a3efca..55c1069 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java @@ -1,62 +1,62 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.typeinfo; - -import java.io.Serializable; - -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; - -/** - * Stores information about a type. Always use the TypeInfoFactory to create new - * TypeInfo objects. - * - * We support 5 categories of types: 1. Primitive objects (String, Number, etc) - * 2. List objects (a list of objects of a single type) 3. Map objects (a map - * from objects of one type to objects of another type) 4. Struct objects (a - * list of fields with names and their own types) 5. Union objects - */ -public abstract class TypeInfo implements Serializable { - - private static final long serialVersionUID = 1L; - - protected TypeInfo() { - } - - /** - * The Category of this TypeInfo. Possible values are Primitive, List, Map, - * Struct and Union, which corresponds to the 5 sub-classes of TypeInfo. - */ - public abstract Category getCategory(); - - /** - * A String representation of the TypeInfo. - */ - public abstract String getTypeName(); - - @Override - public String toString() { - return getTypeName(); - } - - @Override - public abstract boolean equals(Object o); - - @Override - public abstract int hashCode(); -} +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.typeinfo; + +import java.io.Serializable; + +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; + +/** + * Stores information about a type. Always use the TypeInfoFactory to create new + * TypeInfo objects. + * + * We support 5 categories of types: 1. Primitive objects (String, Number, etc) + * 2. List objects (a list of objects of a single type) 3. Map objects (a map + * from objects of one type to objects of another type) 4. Struct objects (a + * list of fields with names and their own types) 5. Union objects + */ +public abstract class TypeInfo implements Serializable { + + private static final long serialVersionUID = 1L; + + protected TypeInfo() { + } + + /** + * The Category of this TypeInfo. Possible values are Primitive, List, Map, + * Struct and Union, which corresponds to the 5 sub-classes of TypeInfo. + */ + public abstract Category getCategory(); + + /** + * A String representation of the TypeInfo. + */ + public abstract String getTypeName(); + + @Override + public String toString() { + return getTypeName(); + } + + @Override + public abstract boolean equals(Object o); + + @Override + public abstract int hashCode(); +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java index 4f9fa75..1f0a578 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java @@ -1,141 +1,141 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.typeinfo; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; - -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; - -/** - * TypeInfoFactory can be used to create the TypeInfo object for any types. - * - * TypeInfo objects are all read-only so we can reuse them easily. - * TypeInfoFactory has internal cache to make sure we don't create 2 TypeInfo - * objects that represents the same type. - */ -public final class TypeInfoFactory { - - static HashMap cachedPrimitiveTypeInfo = new HashMap(); - - private TypeInfoFactory() { - // prevent instantiation - } - - public static TypeInfo getPrimitiveTypeInfo(String typeName) { - if (null == PrimitiveObjectInspectorUtils - .getTypeEntryFromTypeName(typeName)) { - throw new RuntimeException("Cannot getPrimitiveTypeInfo for " + typeName); - } - TypeInfo result = cachedPrimitiveTypeInfo.get(typeName); - if (result == null) { - result = new PrimitiveTypeInfo(typeName); - cachedPrimitiveTypeInfo.put(typeName, result); - } - return result; - } - - public static final TypeInfo voidTypeInfo = getPrimitiveTypeInfo(serdeConstants.VOID_TYPE_NAME); - public static final TypeInfo booleanTypeInfo = getPrimitiveTypeInfo(serdeConstants.BOOLEAN_TYPE_NAME); - public static final TypeInfo intTypeInfo = getPrimitiveTypeInfo(serdeConstants.INT_TYPE_NAME); - public static final TypeInfo longTypeInfo = getPrimitiveTypeInfo(serdeConstants.BIGINT_TYPE_NAME); - public static final TypeInfo stringTypeInfo = getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME); - public static final TypeInfo floatTypeInfo = getPrimitiveTypeInfo(serdeConstants.FLOAT_TYPE_NAME); - public static final TypeInfo doubleTypeInfo = getPrimitiveTypeInfo(serdeConstants.DOUBLE_TYPE_NAME); - public static final TypeInfo byteTypeInfo = getPrimitiveTypeInfo(serdeConstants.TINYINT_TYPE_NAME); - public static final TypeInfo shortTypeInfo = getPrimitiveTypeInfo(serdeConstants.SMALLINT_TYPE_NAME); - public static final TypeInfo timestampTypeInfo = getPrimitiveTypeInfo(serdeConstants.TIMESTAMP_TYPE_NAME); - public static final TypeInfo binaryTypeInfo = getPrimitiveTypeInfo(serdeConstants.BINARY_TYPE_NAME); - - public static final TypeInfo unknownTypeInfo = getPrimitiveTypeInfo("unknown"); - - public static TypeInfo getPrimitiveTypeInfoFromPrimitiveWritable( - Class clazz) { - String typeName = PrimitiveObjectInspectorUtils - .getTypeNameFromPrimitiveWritable(clazz); - if (typeName == null) { - throw new RuntimeException("Internal error: Cannot get typeName for " - + clazz); - } - return getPrimitiveTypeInfo(typeName); - } - - public static TypeInfo getPrimitiveTypeInfoFromJavaPrimitive(Class clazz) { - return getPrimitiveTypeInfo(PrimitiveObjectInspectorUtils - .getTypeNameFromPrimitiveJava(clazz)); - } - - static HashMap>, TypeInfo> cachedStructTypeInfo = - new HashMap>, TypeInfo>(); - - public static TypeInfo getStructTypeInfo(List names, - List typeInfos) { - ArrayList> signature = new ArrayList>(2); - signature.add(names); - signature.add(typeInfos); - TypeInfo result = cachedStructTypeInfo.get(signature); - if (result == null) { - result = new StructTypeInfo(names, typeInfos); - cachedStructTypeInfo.put(signature, result); - } - return result; - } - - static HashMap, TypeInfo> cachedUnionTypeInfo = - new HashMap, TypeInfo>(); - - public static TypeInfo getUnionTypeInfo(List typeInfos) { - TypeInfo result = cachedUnionTypeInfo.get(typeInfos); - if (result == null) { - result = new UnionTypeInfo(typeInfos); - cachedUnionTypeInfo.put(typeInfos, result); - } - return result; - } - - static HashMap cachedListTypeInfo = new HashMap(); - - public static TypeInfo getListTypeInfo(TypeInfo elementTypeInfo) { - TypeInfo result = cachedListTypeInfo.get(elementTypeInfo); - if (result == null) { - result = new ListTypeInfo(elementTypeInfo); - cachedListTypeInfo.put(elementTypeInfo, result); - } - return result; - } - - static HashMap, TypeInfo> cachedMapTypeInfo = - new HashMap, TypeInfo>(); - - public static TypeInfo getMapTypeInfo(TypeInfo keyTypeInfo, - TypeInfo valueTypeInfo) { - ArrayList signature = new ArrayList(2); - signature.add(keyTypeInfo); - signature.add(valueTypeInfo); - TypeInfo result = cachedMapTypeInfo.get(signature); - if (result == null) { - result = new MapTypeInfo(keyTypeInfo, valueTypeInfo); - cachedMapTypeInfo.put(signature, result); - } - return result; - }; - -} +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.typeinfo; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; + +/** + * TypeInfoFactory can be used to create the TypeInfo object for any types. + * + * TypeInfo objects are all read-only so we can reuse them easily. + * TypeInfoFactory has internal cache to make sure we don't create 2 TypeInfo + * objects that represents the same type. + */ +public final class TypeInfoFactory { + + static HashMap cachedPrimitiveTypeInfo = new HashMap(); + + private TypeInfoFactory() { + // prevent instantiation + } + + public static TypeInfo getPrimitiveTypeInfo(String typeName) { + if (null == PrimitiveObjectInspectorUtils + .getTypeEntryFromTypeName(typeName)) { + throw new RuntimeException("Cannot getPrimitiveTypeInfo for " + typeName); + } + TypeInfo result = cachedPrimitiveTypeInfo.get(typeName); + if (result == null) { + result = new PrimitiveTypeInfo(typeName); + cachedPrimitiveTypeInfo.put(typeName, result); + } + return result; + } + + public static final TypeInfo voidTypeInfo = getPrimitiveTypeInfo(serdeConstants.VOID_TYPE_NAME); + public static final TypeInfo booleanTypeInfo = getPrimitiveTypeInfo(serdeConstants.BOOLEAN_TYPE_NAME); + public static final TypeInfo intTypeInfo = getPrimitiveTypeInfo(serdeConstants.INT_TYPE_NAME); + public static final TypeInfo longTypeInfo = getPrimitiveTypeInfo(serdeConstants.BIGINT_TYPE_NAME); + public static final TypeInfo stringTypeInfo = getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME); + public static final TypeInfo floatTypeInfo = getPrimitiveTypeInfo(serdeConstants.FLOAT_TYPE_NAME); + public static final TypeInfo doubleTypeInfo = getPrimitiveTypeInfo(serdeConstants.DOUBLE_TYPE_NAME); + public static final TypeInfo byteTypeInfo = getPrimitiveTypeInfo(serdeConstants.TINYINT_TYPE_NAME); + public static final TypeInfo shortTypeInfo = getPrimitiveTypeInfo(serdeConstants.SMALLINT_TYPE_NAME); + public static final TypeInfo timestampTypeInfo = getPrimitiveTypeInfo(serdeConstants.TIMESTAMP_TYPE_NAME); + public static final TypeInfo binaryTypeInfo = getPrimitiveTypeInfo(serdeConstants.BINARY_TYPE_NAME); + + public static final TypeInfo unknownTypeInfo = getPrimitiveTypeInfo("unknown"); + + public static TypeInfo getPrimitiveTypeInfoFromPrimitiveWritable( + Class clazz) { + String typeName = PrimitiveObjectInspectorUtils + .getTypeNameFromPrimitiveWritable(clazz); + if (typeName == null) { + throw new RuntimeException("Internal error: Cannot get typeName for " + + clazz); + } + return getPrimitiveTypeInfo(typeName); + } + + public static TypeInfo getPrimitiveTypeInfoFromJavaPrimitive(Class clazz) { + return getPrimitiveTypeInfo(PrimitiveObjectInspectorUtils + .getTypeNameFromPrimitiveJava(clazz)); + } + + static HashMap>, TypeInfo> cachedStructTypeInfo = + new HashMap>, TypeInfo>(); + + public static TypeInfo getStructTypeInfo(List names, + List typeInfos) { + ArrayList> signature = new ArrayList>(2); + signature.add(names); + signature.add(typeInfos); + TypeInfo result = cachedStructTypeInfo.get(signature); + if (result == null) { + result = new StructTypeInfo(names, typeInfos); + cachedStructTypeInfo.put(signature, result); + } + return result; + } + + static HashMap, TypeInfo> cachedUnionTypeInfo = + new HashMap, TypeInfo>(); + + public static TypeInfo getUnionTypeInfo(List typeInfos) { + TypeInfo result = cachedUnionTypeInfo.get(typeInfos); + if (result == null) { + result = new UnionTypeInfo(typeInfos); + cachedUnionTypeInfo.put(typeInfos, result); + } + return result; + } + + static HashMap cachedListTypeInfo = new HashMap(); + + public static TypeInfo getListTypeInfo(TypeInfo elementTypeInfo) { + TypeInfo result = cachedListTypeInfo.get(elementTypeInfo); + if (result == null) { + result = new ListTypeInfo(elementTypeInfo); + cachedListTypeInfo.put(elementTypeInfo, result); + } + return result; + } + + static HashMap, TypeInfo> cachedMapTypeInfo = + new HashMap, TypeInfo>(); + + public static TypeInfo getMapTypeInfo(TypeInfo keyTypeInfo, + TypeInfo valueTypeInfo) { + ArrayList signature = new ArrayList(2); + signature.add(keyTypeInfo); + signature.add(valueTypeInfo); + TypeInfo result = cachedMapTypeInfo.get(signature); + if (result == null) { + result = new MapTypeInfo(keyTypeInfo, valueTypeInfo); + cachedMapTypeInfo.put(signature, result); + } + return result; + }; + +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java index 1193fd4..af68217 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java @@ -1,633 +1,633 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.serde2.typeinfo; - -import java.lang.reflect.Field; -import java.lang.reflect.GenericArrayType; -import java.lang.reflect.Method; -import java.lang.reflect.ParameterizedType; -import java.lang.reflect.Type; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; - -/** - * TypeInfoUtils. - * - */ -public final class TypeInfoUtils { - - private TypeInfoUtils() { - // prevent instantiation - } - - /** - * Return the extended TypeInfo from a Java type. By extended TypeInfo, we - * allow unknownType for java.lang.Object. - * - * @param t - * The Java type. - * @param m - * The method, only used for generating error messages. - */ - private static TypeInfo getExtendedTypeInfoFromJavaType(Type t, Method m) { - - if (t == Object.class) { - return TypeInfoFactory.unknownTypeInfo; - } - - if (t instanceof ParameterizedType) { - ParameterizedType pt = (ParameterizedType) t; - // List? - if (List.class == (Class) pt.getRawType() - || ArrayList.class == (Class) pt.getRawType()) { - return TypeInfoFactory.getListTypeInfo(getExtendedTypeInfoFromJavaType( - pt.getActualTypeArguments()[0], m)); - } - // Map? - if (Map.class == (Class) pt.getRawType() - || HashMap.class == (Class) pt.getRawType()) { - return TypeInfoFactory.getMapTypeInfo(getExtendedTypeInfoFromJavaType( - pt.getActualTypeArguments()[0], m), - getExtendedTypeInfoFromJavaType(pt.getActualTypeArguments()[1], m)); - } - // Otherwise convert t to RawType so we will fall into the following if - // block. - t = pt.getRawType(); - } - - // Must be a class. - if (!(t instanceof Class)) { - throw new RuntimeException("Hive does not understand type " + t - + " from " + m); - } - Class c = (Class) t; - - // Java Primitive Type? - if (PrimitiveObjectInspectorUtils.isPrimitiveJavaType(c)) { - return TypeInfoUtils - .getTypeInfoFromObjectInspector(PrimitiveObjectInspectorFactory - .getPrimitiveJavaObjectInspector(PrimitiveObjectInspectorUtils - .getTypeEntryFromPrimitiveJavaType(c).primitiveCategory)); - } - - // Java Primitive Class? - if (PrimitiveObjectInspectorUtils.isPrimitiveJavaClass(c)) { - return TypeInfoUtils - .getTypeInfoFromObjectInspector(PrimitiveObjectInspectorFactory - .getPrimitiveJavaObjectInspector(PrimitiveObjectInspectorUtils - .getTypeEntryFromPrimitiveJavaClass(c).primitiveCategory)); - } - - // Primitive Writable class? - if (PrimitiveObjectInspectorUtils.isPrimitiveWritableClass(c)) { - return TypeInfoUtils - .getTypeInfoFromObjectInspector(PrimitiveObjectInspectorFactory - .getPrimitiveWritableObjectInspector(PrimitiveObjectInspectorUtils - .getTypeEntryFromPrimitiveWritableClass(c).primitiveCategory)); - } - - // Must be a struct - Field[] fields = ObjectInspectorUtils.getDeclaredNonStaticFields(c); - ArrayList fieldNames = new ArrayList(fields.length); - ArrayList fieldTypeInfos = new ArrayList(fields.length); - for (Field field : fields) { - fieldNames.add(field.getName()); - fieldTypeInfos.add(getExtendedTypeInfoFromJavaType( - field.getGenericType(), m)); - } - return TypeInfoFactory.getStructTypeInfo(fieldNames, fieldTypeInfos); - } - - /** - * Returns the array element type, if the Type is an array (Object[]), or - * GenericArrayType (Map[]). Otherwise return null. - */ - public static Type getArrayElementType(Type t) { - if (t instanceof Class && ((Class) t).isArray()) { - Class arrayClass = (Class) t; - return arrayClass.getComponentType(); - } else if (t instanceof GenericArrayType) { - GenericArrayType arrayType = (GenericArrayType) t; - return arrayType.getGenericComponentType(); - } - return null; - } - - /** - * Get the parameter TypeInfo for a method. - * - * @param size - * In case the last parameter of Method is an array, we will try to - * return a List with the specified size by repeating the - * element of the array at the end. In case the size is smaller than - * the minimum possible number of arguments for the method, null will - * be returned. - */ - public static List getParameterTypeInfos(Method m, int size) { - Type[] methodParameterTypes = m.getGenericParameterTypes(); - - // Whether the method takes variable-length arguments - // Whether the method takes an array like Object[], - // or String[] etc in the last argument. - Type lastParaElementType = TypeInfoUtils - .getArrayElementType(methodParameterTypes.length == 0 ? null - : methodParameterTypes[methodParameterTypes.length - 1]); - boolean isVariableLengthArgument = (lastParaElementType != null); - - List typeInfos = null; - if (!isVariableLengthArgument) { - // Normal case, no variable-length arguments - if (size != methodParameterTypes.length) { - return null; - } - typeInfos = new ArrayList(methodParameterTypes.length); - for (Type methodParameterType : methodParameterTypes) { - typeInfos.add(getExtendedTypeInfoFromJavaType(methodParameterType, m)); - } - } else { - // Variable-length arguments - if (size < methodParameterTypes.length - 1) { - return null; - } - typeInfos = new ArrayList(size); - for (int i = 0; i < methodParameterTypes.length - 1; i++) { - typeInfos.add(getExtendedTypeInfoFromJavaType(methodParameterTypes[i], - m)); - } - for (int i = methodParameterTypes.length - 1; i < size; i++) { - typeInfos.add(getExtendedTypeInfoFromJavaType(lastParaElementType, m)); - } - } - return typeInfos; - } - - /** - * Parse a recursive TypeInfo list String. For example, the following inputs - * are valid inputs: - * "int,string,map,list>>,list>" - * The separators between TypeInfos can be ",", ":", or ";". - * - * In order to use this class: TypeInfoParser parser = new - * TypeInfoParser("int,string"); ArrayList typeInfos = - * parser.parseTypeInfos(); - */ - private static class TypeInfoParser { - - private static class Token { - public int position; - public String text; - public boolean isType; - - @Override - public String toString() { - return "" + position + ":" + text; - } - }; - - private static boolean isTypeChar(char c) { - return Character.isLetterOrDigit(c) || c == '_' || c == '.'; - } - - /** - * Tokenize the typeInfoString. The rule is simple: all consecutive - * alphadigits and '_', '.' are in one token, and all other characters are - * one character per token. - * - * tokenize("map") should return - * ["map","<","int",",","string",">"] - */ - private static ArrayList tokenize(String typeInfoString) { - ArrayList tokens = new ArrayList(0); - int begin = 0; - int end = 1; - while (end <= typeInfoString.length()) { - // last character ends a token? - if (end == typeInfoString.length() - || !isTypeChar(typeInfoString.charAt(end - 1)) - || !isTypeChar(typeInfoString.charAt(end))) { - Token t = new Token(); - t.position = begin; - t.text = typeInfoString.substring(begin, end); - t.isType = isTypeChar(typeInfoString.charAt(begin)); - tokens.add(t); - begin = end; - } - end++; - } - return tokens; - } - - public TypeInfoParser(String typeInfoString) { - this.typeInfoString = typeInfoString; - typeInfoTokens = tokenize(typeInfoString); - } - - private final String typeInfoString; - private final ArrayList typeInfoTokens; - private ArrayList typeInfos; - private int iToken; - - public ArrayList parseTypeInfos() { - typeInfos = new ArrayList(); - iToken = 0; - while (iToken < typeInfoTokens.size()) { - typeInfos.add(parseType()); - if (iToken < typeInfoTokens.size()) { - Token separator = typeInfoTokens.get(iToken); - if (",".equals(separator.text) || ";".equals(separator.text) - || ":".equals(separator.text)) { - iToken++; - } else { - throw new IllegalArgumentException( - "Error: ',', ':', or ';' expected at position " - + separator.position + " from '" + typeInfoString + "' " - + typeInfoTokens); - } - } - } - return typeInfos; - } - - private Token expect(String item) { - return expect(item, null); - } - - private Token expect(String item, String alternative) { - if (iToken >= typeInfoTokens.size()) { - throw new IllegalArgumentException("Error: " + item - + " expected at the end of '" + typeInfoString + "'"); - } - Token t = typeInfoTokens.get(iToken); - if (item.equals("type")) { - if (!serdeConstants.LIST_TYPE_NAME.equals(t.text) - && !serdeConstants.MAP_TYPE_NAME.equals(t.text) - && !serdeConstants.STRUCT_TYPE_NAME.equals(t.text) - && !serdeConstants.UNION_TYPE_NAME.equals(t.text) - && null == PrimitiveObjectInspectorUtils - .getTypeEntryFromTypeName(t.text) - && !t.text.equals(alternative)) { - throw new IllegalArgumentException("Error: " + item - + " expected at the position " + t.position + " of '" - + typeInfoString + "' but '" + t.text + "' is found."); - } - } else if (item.equals("name")) { - if (!t.isType && !t.text.equals(alternative)) { - throw new IllegalArgumentException("Error: " + item - + " expected at the position " + t.position + " of '" - + typeInfoString + "' but '" + t.text + "' is found."); - } - } else { - if (!item.equals(t.text) && !t.text.equals(alternative)) { - throw new IllegalArgumentException("Error: " + item - + " expected at the position " + t.position + " of '" - + typeInfoString + "' but '" + t.text + "' is found."); - } - } - iToken++; - return t; - } - - private TypeInfo parseType() { - - Token t = expect("type"); - - // Is this a primitive type? - PrimitiveTypeEntry primitiveType = PrimitiveObjectInspectorUtils - .getTypeEntryFromTypeName(t.text); - if (primitiveType != null - && !primitiveType.primitiveCategory.equals(PrimitiveCategory.UNKNOWN)) { - return TypeInfoFactory.getPrimitiveTypeInfo(primitiveType.typeName); - } - - // Is this a list type? - if (serdeConstants.LIST_TYPE_NAME.equals(t.text)) { - expect("<"); - TypeInfo listElementType = parseType(); - expect(">"); - return TypeInfoFactory.getListTypeInfo(listElementType); - } - - // Is this a map type? - if (serdeConstants.MAP_TYPE_NAME.equals(t.text)) { - expect("<"); - TypeInfo mapKeyType = parseType(); - expect(","); - TypeInfo mapValueType = parseType(); - expect(">"); - return TypeInfoFactory.getMapTypeInfo(mapKeyType, mapValueType); - } - - // Is this a struct type? - if (serdeConstants.STRUCT_TYPE_NAME.equals(t.text)) { - ArrayList fieldNames = new ArrayList(); - ArrayList fieldTypeInfos = new ArrayList(); - boolean first = true; - do { - if (first) { - expect("<"); - first = false; - } else { - Token separator = expect(">", ","); - if (separator.text.equals(">")) { - // end of struct - break; - } - } - Token name = expect("name"); - fieldNames.add(name.text); - expect(":"); - fieldTypeInfos.add(parseType()); - } while (true); - - return TypeInfoFactory.getStructTypeInfo(fieldNames, fieldTypeInfos); - } - // Is this a union type? - if (serdeConstants.UNION_TYPE_NAME.equals(t.text)) { - List objectTypeInfos = new ArrayList(); - boolean first = true; - do { - if (first) { - expect("<"); - first = false; - } else { - Token separator = expect(">", ","); - if (separator.text.equals(">")) { - // end of union - break; - } - } - objectTypeInfos.add(parseType()); - } while (true); - - return TypeInfoFactory.getUnionTypeInfo(objectTypeInfos); - } - - throw new RuntimeException("Internal error parsing position " - + t.position + " of '" + typeInfoString + "'"); - } - - } - - static HashMap cachedStandardObjectInspector = - new HashMap(); - - /** - * Returns the standard object inspector that can be used to translate an - * object of that typeInfo to a standard object type. - */ - public static ObjectInspector getStandardWritableObjectInspectorFromTypeInfo( - TypeInfo typeInfo) { - ObjectInspector result = cachedStandardObjectInspector.get(typeInfo); - if (result == null) { - switch (typeInfo.getCategory()) { - case PRIMITIVE: { - result = PrimitiveObjectInspectorFactory - .getPrimitiveWritableObjectInspector(((PrimitiveTypeInfo) typeInfo) - .getPrimitiveCategory()); - break; - } - case LIST: { - ObjectInspector elementObjectInspector = - getStandardWritableObjectInspectorFromTypeInfo(((ListTypeInfo) typeInfo) - .getListElementTypeInfo()); - result = ObjectInspectorFactory - .getStandardListObjectInspector(elementObjectInspector); - break; - } - case MAP: { - MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; - ObjectInspector keyObjectInspector = - getStandardWritableObjectInspectorFromTypeInfo(mapTypeInfo.getMapKeyTypeInfo()); - ObjectInspector valueObjectInspector = - getStandardWritableObjectInspectorFromTypeInfo(mapTypeInfo.getMapValueTypeInfo()); - result = ObjectInspectorFactory.getStandardMapObjectInspector( - keyObjectInspector, valueObjectInspector); - break; - } - case STRUCT: { - StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; - List fieldNames = structTypeInfo.getAllStructFieldNames(); - List fieldTypeInfos = structTypeInfo - .getAllStructFieldTypeInfos(); - List fieldObjectInspectors = new ArrayList( - fieldTypeInfos.size()); - for (int i = 0; i < fieldTypeInfos.size(); i++) { - fieldObjectInspectors - .add(getStandardWritableObjectInspectorFromTypeInfo(fieldTypeInfos - .get(i))); - } - result = ObjectInspectorFactory.getStandardStructObjectInspector( - fieldNames, fieldObjectInspectors); - break; - } - case UNION: { - UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo; - List objectTypeInfos = unionTypeInfo - .getAllUnionObjectTypeInfos(); - List fieldObjectInspectors = - new ArrayList(objectTypeInfos.size()); - for (int i = 0; i < objectTypeInfos.size(); i++) { - fieldObjectInspectors - .add(getStandardWritableObjectInspectorFromTypeInfo(objectTypeInfos - .get(i))); - } - result = ObjectInspectorFactory.getStandardUnionObjectInspector( - fieldObjectInspectors); - break; - } - - default: { - result = null; - } - } - cachedStandardObjectInspector.put(typeInfo, result); - } - return result; - } - - static HashMap cachedStandardJavaObjectInspector = - new HashMap(); - - /** - * Returns the standard object inspector that can be used to translate an - * object of that typeInfo to a standard object type. - */ - public static ObjectInspector getStandardJavaObjectInspectorFromTypeInfo( - TypeInfo typeInfo) { - ObjectInspector result = cachedStandardJavaObjectInspector.get(typeInfo); - if (result == null) { - switch (typeInfo.getCategory()) { - case PRIMITIVE: { - // NOTE: we use JavaPrimitiveObjectInspector instead of - // StandardPrimitiveObjectInspector - result = PrimitiveObjectInspectorFactory - .getPrimitiveJavaObjectInspector(PrimitiveObjectInspectorUtils - .getTypeEntryFromTypeName(typeInfo.getTypeName()).primitiveCategory); - break; - } - case LIST: { - ObjectInspector elementObjectInspector = - getStandardJavaObjectInspectorFromTypeInfo(((ListTypeInfo) typeInfo) - .getListElementTypeInfo()); - result = ObjectInspectorFactory - .getStandardListObjectInspector(elementObjectInspector); - break; - } - case MAP: { - MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; - ObjectInspector keyObjectInspector = getStandardJavaObjectInspectorFromTypeInfo(mapTypeInfo - .getMapKeyTypeInfo()); - ObjectInspector valueObjectInspector = - getStandardJavaObjectInspectorFromTypeInfo(mapTypeInfo.getMapValueTypeInfo()); - result = ObjectInspectorFactory.getStandardMapObjectInspector( - keyObjectInspector, valueObjectInspector); - break; - } - case STRUCT: { - StructTypeInfo strucTypeInfo = (StructTypeInfo) typeInfo; - List fieldNames = strucTypeInfo.getAllStructFieldNames(); - List fieldTypeInfos = strucTypeInfo - .getAllStructFieldTypeInfos(); - List fieldObjectInspectors = new ArrayList( - fieldTypeInfos.size()); - for (int i = 0; i < fieldTypeInfos.size(); i++) { - fieldObjectInspectors - .add(getStandardJavaObjectInspectorFromTypeInfo(fieldTypeInfos - .get(i))); - } - result = ObjectInspectorFactory.getStandardStructObjectInspector( - fieldNames, fieldObjectInspectors); - break; - } - case UNION: { - UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo; - List objectTypeInfos = unionTypeInfo - .getAllUnionObjectTypeInfos(); - List fieldObjectInspectors = - new ArrayList(objectTypeInfos.size()); - for (int i = 0; i < objectTypeInfos.size(); i++) { - fieldObjectInspectors - .add(getStandardJavaObjectInspectorFromTypeInfo(objectTypeInfos - .get(i))); - } - result = ObjectInspectorFactory.getStandardUnionObjectInspector( - fieldObjectInspectors); - break; - } - default: { - result = null; - } - } - cachedStandardJavaObjectInspector.put(typeInfo, result); - } - return result; - } - - /** - * Get the TypeInfo object from the ObjectInspector object by recursively - * going into the ObjectInspector structure. - */ - public static TypeInfo getTypeInfoFromObjectInspector(ObjectInspector oi) { - // OPTIMIZATION for later. - // if (oi instanceof TypeInfoBasedObjectInspector) { - // TypeInfoBasedObjectInspector typeInfoBasedObjectInspector = - // (ObjectInspector)oi; - // return typeInfoBasedObjectInspector.getTypeInfo(); - // } - if (oi == null) { - return null; - } - - // Recursively going into ObjectInspector structure - TypeInfo result = null; - switch (oi.getCategory()) { - case PRIMITIVE: { - PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; - result = TypeInfoFactory.getPrimitiveTypeInfo(poi.getTypeName()); - break; - } - case LIST: { - ListObjectInspector loi = (ListObjectInspector) oi; - result = TypeInfoFactory - .getListTypeInfo(getTypeInfoFromObjectInspector(loi - .getListElementObjectInspector())); - break; - } - case MAP: { - MapObjectInspector moi = (MapObjectInspector) oi; - result = TypeInfoFactory.getMapTypeInfo( - getTypeInfoFromObjectInspector(moi.getMapKeyObjectInspector()), - getTypeInfoFromObjectInspector(moi.getMapValueObjectInspector())); - break; - } - case STRUCT: { - StructObjectInspector soi = (StructObjectInspector) oi; - List fields = soi.getAllStructFieldRefs(); - List fieldNames = new ArrayList(fields.size()); - List fieldTypeInfos = new ArrayList(fields.size()); - for (StructField f : fields) { - fieldNames.add(f.getFieldName()); - fieldTypeInfos.add(getTypeInfoFromObjectInspector(f - .getFieldObjectInspector())); - } - result = TypeInfoFactory.getStructTypeInfo(fieldNames, fieldTypeInfos); - break; - } - case UNION: { - UnionObjectInspector uoi = (UnionObjectInspector) oi; - List objectTypeInfos = new ArrayList(); - for (ObjectInspector eoi : uoi.getObjectInspectors()) { - objectTypeInfos.add(getTypeInfoFromObjectInspector(eoi)); - } - result = TypeInfoFactory.getUnionTypeInfo(objectTypeInfos); - break; - } - default: { - throw new RuntimeException("Unknown ObjectInspector category!"); - } - } - return result; - } - - public static ArrayList getTypeInfosFromTypeString(String typeString) { - TypeInfoParser parser = new TypeInfoParser(typeString); - return parser.parseTypeInfos(); - } - - public static TypeInfo getTypeInfoFromTypeString(String typeString) { - TypeInfoParser parser = new TypeInfoParser(typeString); - return parser.parseTypeInfos().get(0); - } -} +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.typeinfo; + +import java.lang.reflect.Field; +import java.lang.reflect.GenericArrayType; +import java.lang.reflect.Method; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; + +/** + * TypeInfoUtils. + * + */ +public final class TypeInfoUtils { + + private TypeInfoUtils() { + // prevent instantiation + } + + /** + * Return the extended TypeInfo from a Java type. By extended TypeInfo, we + * allow unknownType for java.lang.Object. + * + * @param t + * The Java type. + * @param m + * The method, only used for generating error messages. + */ + private static TypeInfo getExtendedTypeInfoFromJavaType(Type t, Method m) { + + if (t == Object.class) { + return TypeInfoFactory.unknownTypeInfo; + } + + if (t instanceof ParameterizedType) { + ParameterizedType pt = (ParameterizedType) t; + // List? + if (List.class == (Class) pt.getRawType() + || ArrayList.class == (Class) pt.getRawType()) { + return TypeInfoFactory.getListTypeInfo(getExtendedTypeInfoFromJavaType( + pt.getActualTypeArguments()[0], m)); + } + // Map? + if (Map.class == (Class) pt.getRawType() + || HashMap.class == (Class) pt.getRawType()) { + return TypeInfoFactory.getMapTypeInfo(getExtendedTypeInfoFromJavaType( + pt.getActualTypeArguments()[0], m), + getExtendedTypeInfoFromJavaType(pt.getActualTypeArguments()[1], m)); + } + // Otherwise convert t to RawType so we will fall into the following if + // block. + t = pt.getRawType(); + } + + // Must be a class. + if (!(t instanceof Class)) { + throw new RuntimeException("Hive does not understand type " + t + + " from " + m); + } + Class c = (Class) t; + + // Java Primitive Type? + if (PrimitiveObjectInspectorUtils.isPrimitiveJavaType(c)) { + return TypeInfoUtils + .getTypeInfoFromObjectInspector(PrimitiveObjectInspectorFactory + .getPrimitiveJavaObjectInspector(PrimitiveObjectInspectorUtils + .getTypeEntryFromPrimitiveJavaType(c).primitiveCategory)); + } + + // Java Primitive Class? + if (PrimitiveObjectInspectorUtils.isPrimitiveJavaClass(c)) { + return TypeInfoUtils + .getTypeInfoFromObjectInspector(PrimitiveObjectInspectorFactory + .getPrimitiveJavaObjectInspector(PrimitiveObjectInspectorUtils + .getTypeEntryFromPrimitiveJavaClass(c).primitiveCategory)); + } + + // Primitive Writable class? + if (PrimitiveObjectInspectorUtils.isPrimitiveWritableClass(c)) { + return TypeInfoUtils + .getTypeInfoFromObjectInspector(PrimitiveObjectInspectorFactory + .getPrimitiveWritableObjectInspector(PrimitiveObjectInspectorUtils + .getTypeEntryFromPrimitiveWritableClass(c).primitiveCategory)); + } + + // Must be a struct + Field[] fields = ObjectInspectorUtils.getDeclaredNonStaticFields(c); + ArrayList fieldNames = new ArrayList(fields.length); + ArrayList fieldTypeInfos = new ArrayList(fields.length); + for (Field field : fields) { + fieldNames.add(field.getName()); + fieldTypeInfos.add(getExtendedTypeInfoFromJavaType( + field.getGenericType(), m)); + } + return TypeInfoFactory.getStructTypeInfo(fieldNames, fieldTypeInfos); + } + + /** + * Returns the array element type, if the Type is an array (Object[]), or + * GenericArrayType (Map[]). Otherwise return null. + */ + public static Type getArrayElementType(Type t) { + if (t instanceof Class && ((Class) t).isArray()) { + Class arrayClass = (Class) t; + return arrayClass.getComponentType(); + } else if (t instanceof GenericArrayType) { + GenericArrayType arrayType = (GenericArrayType) t; + return arrayType.getGenericComponentType(); + } + return null; + } + + /** + * Get the parameter TypeInfo for a method. + * + * @param size + * In case the last parameter of Method is an array, we will try to + * return a List with the specified size by repeating the + * element of the array at the end. In case the size is smaller than + * the minimum possible number of arguments for the method, null will + * be returned. + */ + public static List getParameterTypeInfos(Method m, int size) { + Type[] methodParameterTypes = m.getGenericParameterTypes(); + + // Whether the method takes variable-length arguments + // Whether the method takes an array like Object[], + // or String[] etc in the last argument. + Type lastParaElementType = TypeInfoUtils + .getArrayElementType(methodParameterTypes.length == 0 ? null + : methodParameterTypes[methodParameterTypes.length - 1]); + boolean isVariableLengthArgument = (lastParaElementType != null); + + List typeInfos = null; + if (!isVariableLengthArgument) { + // Normal case, no variable-length arguments + if (size != methodParameterTypes.length) { + return null; + } + typeInfos = new ArrayList(methodParameterTypes.length); + for (Type methodParameterType : methodParameterTypes) { + typeInfos.add(getExtendedTypeInfoFromJavaType(methodParameterType, m)); + } + } else { + // Variable-length arguments + if (size < methodParameterTypes.length - 1) { + return null; + } + typeInfos = new ArrayList(size); + for (int i = 0; i < methodParameterTypes.length - 1; i++) { + typeInfos.add(getExtendedTypeInfoFromJavaType(methodParameterTypes[i], + m)); + } + for (int i = methodParameterTypes.length - 1; i < size; i++) { + typeInfos.add(getExtendedTypeInfoFromJavaType(lastParaElementType, m)); + } + } + return typeInfos; + } + + /** + * Parse a recursive TypeInfo list String. For example, the following inputs + * are valid inputs: + * "int,string,map,list>>,list>" + * The separators between TypeInfos can be ",", ":", or ";". + * + * In order to use this class: TypeInfoParser parser = new + * TypeInfoParser("int,string"); ArrayList typeInfos = + * parser.parseTypeInfos(); + */ + private static class TypeInfoParser { + + private static class Token { + public int position; + public String text; + public boolean isType; + + @Override + public String toString() { + return "" + position + ":" + text; + } + }; + + private static boolean isTypeChar(char c) { + return Character.isLetterOrDigit(c) || c == '_' || c == '.'; + } + + /** + * Tokenize the typeInfoString. The rule is simple: all consecutive + * alphadigits and '_', '.' are in one token, and all other characters are + * one character per token. + * + * tokenize("map") should return + * ["map","<","int",",","string",">"] + */ + private static ArrayList tokenize(String typeInfoString) { + ArrayList tokens = new ArrayList(0); + int begin = 0; + int end = 1; + while (end <= typeInfoString.length()) { + // last character ends a token? + if (end == typeInfoString.length() + || !isTypeChar(typeInfoString.charAt(end - 1)) + || !isTypeChar(typeInfoString.charAt(end))) { + Token t = new Token(); + t.position = begin; + t.text = typeInfoString.substring(begin, end); + t.isType = isTypeChar(typeInfoString.charAt(begin)); + tokens.add(t); + begin = end; + } + end++; + } + return tokens; + } + + public TypeInfoParser(String typeInfoString) { + this.typeInfoString = typeInfoString; + typeInfoTokens = tokenize(typeInfoString); + } + + private final String typeInfoString; + private final ArrayList typeInfoTokens; + private ArrayList typeInfos; + private int iToken; + + public ArrayList parseTypeInfos() { + typeInfos = new ArrayList(); + iToken = 0; + while (iToken < typeInfoTokens.size()) { + typeInfos.add(parseType()); + if (iToken < typeInfoTokens.size()) { + Token separator = typeInfoTokens.get(iToken); + if (",".equals(separator.text) || ";".equals(separator.text) + || ":".equals(separator.text)) { + iToken++; + } else { + throw new IllegalArgumentException( + "Error: ',', ':', or ';' expected at position " + + separator.position + " from '" + typeInfoString + "' " + + typeInfoTokens); + } + } + } + return typeInfos; + } + + private Token expect(String item) { + return expect(item, null); + } + + private Token expect(String item, String alternative) { + if (iToken >= typeInfoTokens.size()) { + throw new IllegalArgumentException("Error: " + item + + " expected at the end of '" + typeInfoString + "'"); + } + Token t = typeInfoTokens.get(iToken); + if (item.equals("type")) { + if (!serdeConstants.LIST_TYPE_NAME.equals(t.text) + && !serdeConstants.MAP_TYPE_NAME.equals(t.text) + && !serdeConstants.STRUCT_TYPE_NAME.equals(t.text) + && !serdeConstants.UNION_TYPE_NAME.equals(t.text) + && null == PrimitiveObjectInspectorUtils + .getTypeEntryFromTypeName(t.text) + && !t.text.equals(alternative)) { + throw new IllegalArgumentException("Error: " + item + + " expected at the position " + t.position + " of '" + + typeInfoString + "' but '" + t.text + "' is found."); + } + } else if (item.equals("name")) { + if (!t.isType && !t.text.equals(alternative)) { + throw new IllegalArgumentException("Error: " + item + + " expected at the position " + t.position + " of '" + + typeInfoString + "' but '" + t.text + "' is found."); + } + } else { + if (!item.equals(t.text) && !t.text.equals(alternative)) { + throw new IllegalArgumentException("Error: " + item + + " expected at the position " + t.position + " of '" + + typeInfoString + "' but '" + t.text + "' is found."); + } + } + iToken++; + return t; + } + + private TypeInfo parseType() { + + Token t = expect("type"); + + // Is this a primitive type? + PrimitiveTypeEntry primitiveType = PrimitiveObjectInspectorUtils + .getTypeEntryFromTypeName(t.text); + if (primitiveType != null + && !primitiveType.primitiveCategory.equals(PrimitiveCategory.UNKNOWN)) { + return TypeInfoFactory.getPrimitiveTypeInfo(primitiveType.typeName); + } + + // Is this a list type? + if (serdeConstants.LIST_TYPE_NAME.equals(t.text)) { + expect("<"); + TypeInfo listElementType = parseType(); + expect(">"); + return TypeInfoFactory.getListTypeInfo(listElementType); + } + + // Is this a map type? + if (serdeConstants.MAP_TYPE_NAME.equals(t.text)) { + expect("<"); + TypeInfo mapKeyType = parseType(); + expect(","); + TypeInfo mapValueType = parseType(); + expect(">"); + return TypeInfoFactory.getMapTypeInfo(mapKeyType, mapValueType); + } + + // Is this a struct type? + if (serdeConstants.STRUCT_TYPE_NAME.equals(t.text)) { + ArrayList fieldNames = new ArrayList(); + ArrayList fieldTypeInfos = new ArrayList(); + boolean first = true; + do { + if (first) { + expect("<"); + first = false; + } else { + Token separator = expect(">", ","); + if (separator.text.equals(">")) { + // end of struct + break; + } + } + Token name = expect("name"); + fieldNames.add(name.text); + expect(":"); + fieldTypeInfos.add(parseType()); + } while (true); + + return TypeInfoFactory.getStructTypeInfo(fieldNames, fieldTypeInfos); + } + // Is this a union type? + if (serdeConstants.UNION_TYPE_NAME.equals(t.text)) { + List objectTypeInfos = new ArrayList(); + boolean first = true; + do { + if (first) { + expect("<"); + first = false; + } else { + Token separator = expect(">", ","); + if (separator.text.equals(">")) { + // end of union + break; + } + } + objectTypeInfos.add(parseType()); + } while (true); + + return TypeInfoFactory.getUnionTypeInfo(objectTypeInfos); + } + + throw new RuntimeException("Internal error parsing position " + + t.position + " of '" + typeInfoString + "'"); + } + + } + + static HashMap cachedStandardObjectInspector = + new HashMap(); + + /** + * Returns the standard object inspector that can be used to translate an + * object of that typeInfo to a standard object type. + */ + public static ObjectInspector getStandardWritableObjectInspectorFromTypeInfo( + TypeInfo typeInfo) { + ObjectInspector result = cachedStandardObjectInspector.get(typeInfo); + if (result == null) { + switch (typeInfo.getCategory()) { + case PRIMITIVE: { + result = PrimitiveObjectInspectorFactory + .getPrimitiveWritableObjectInspector(((PrimitiveTypeInfo) typeInfo) + .getPrimitiveCategory()); + break; + } + case LIST: { + ObjectInspector elementObjectInspector = + getStandardWritableObjectInspectorFromTypeInfo(((ListTypeInfo) typeInfo) + .getListElementTypeInfo()); + result = ObjectInspectorFactory + .getStandardListObjectInspector(elementObjectInspector); + break; + } + case MAP: { + MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; + ObjectInspector keyObjectInspector = + getStandardWritableObjectInspectorFromTypeInfo(mapTypeInfo.getMapKeyTypeInfo()); + ObjectInspector valueObjectInspector = + getStandardWritableObjectInspectorFromTypeInfo(mapTypeInfo.getMapValueTypeInfo()); + result = ObjectInspectorFactory.getStandardMapObjectInspector( + keyObjectInspector, valueObjectInspector); + break; + } + case STRUCT: { + StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; + List fieldNames = structTypeInfo.getAllStructFieldNames(); + List fieldTypeInfos = structTypeInfo + .getAllStructFieldTypeInfos(); + List fieldObjectInspectors = new ArrayList( + fieldTypeInfos.size()); + for (int i = 0; i < fieldTypeInfos.size(); i++) { + fieldObjectInspectors + .add(getStandardWritableObjectInspectorFromTypeInfo(fieldTypeInfos + .get(i))); + } + result = ObjectInspectorFactory.getStandardStructObjectInspector( + fieldNames, fieldObjectInspectors); + break; + } + case UNION: { + UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo; + List objectTypeInfos = unionTypeInfo + .getAllUnionObjectTypeInfos(); + List fieldObjectInspectors = + new ArrayList(objectTypeInfos.size()); + for (int i = 0; i < objectTypeInfos.size(); i++) { + fieldObjectInspectors + .add(getStandardWritableObjectInspectorFromTypeInfo(objectTypeInfos + .get(i))); + } + result = ObjectInspectorFactory.getStandardUnionObjectInspector( + fieldObjectInspectors); + break; + } + + default: { + result = null; + } + } + cachedStandardObjectInspector.put(typeInfo, result); + } + return result; + } + + static HashMap cachedStandardJavaObjectInspector = + new HashMap(); + + /** + * Returns the standard object inspector that can be used to translate an + * object of that typeInfo to a standard object type. + */ + public static ObjectInspector getStandardJavaObjectInspectorFromTypeInfo( + TypeInfo typeInfo) { + ObjectInspector result = cachedStandardJavaObjectInspector.get(typeInfo); + if (result == null) { + switch (typeInfo.getCategory()) { + case PRIMITIVE: { + // NOTE: we use JavaPrimitiveObjectInspector instead of + // StandardPrimitiveObjectInspector + result = PrimitiveObjectInspectorFactory + .getPrimitiveJavaObjectInspector(PrimitiveObjectInspectorUtils + .getTypeEntryFromTypeName(typeInfo.getTypeName()).primitiveCategory); + break; + } + case LIST: { + ObjectInspector elementObjectInspector = + getStandardJavaObjectInspectorFromTypeInfo(((ListTypeInfo) typeInfo) + .getListElementTypeInfo()); + result = ObjectInspectorFactory + .getStandardListObjectInspector(elementObjectInspector); + break; + } + case MAP: { + MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; + ObjectInspector keyObjectInspector = getStandardJavaObjectInspectorFromTypeInfo(mapTypeInfo + .getMapKeyTypeInfo()); + ObjectInspector valueObjectInspector = + getStandardJavaObjectInspectorFromTypeInfo(mapTypeInfo.getMapValueTypeInfo()); + result = ObjectInspectorFactory.getStandardMapObjectInspector( + keyObjectInspector, valueObjectInspector); + break; + } + case STRUCT: { + StructTypeInfo strucTypeInfo = (StructTypeInfo) typeInfo; + List fieldNames = strucTypeInfo.getAllStructFieldNames(); + List fieldTypeInfos = strucTypeInfo + .getAllStructFieldTypeInfos(); + List fieldObjectInspectors = new ArrayList( + fieldTypeInfos.size()); + for (int i = 0; i < fieldTypeInfos.size(); i++) { + fieldObjectInspectors + .add(getStandardJavaObjectInspectorFromTypeInfo(fieldTypeInfos + .get(i))); + } + result = ObjectInspectorFactory.getStandardStructObjectInspector( + fieldNames, fieldObjectInspectors); + break; + } + case UNION: { + UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo; + List objectTypeInfos = unionTypeInfo + .getAllUnionObjectTypeInfos(); + List fieldObjectInspectors = + new ArrayList(objectTypeInfos.size()); + for (int i = 0; i < objectTypeInfos.size(); i++) { + fieldObjectInspectors + .add(getStandardJavaObjectInspectorFromTypeInfo(objectTypeInfos + .get(i))); + } + result = ObjectInspectorFactory.getStandardUnionObjectInspector( + fieldObjectInspectors); + break; + } + default: { + result = null; + } + } + cachedStandardJavaObjectInspector.put(typeInfo, result); + } + return result; + } + + /** + * Get the TypeInfo object from the ObjectInspector object by recursively + * going into the ObjectInspector structure. + */ + public static TypeInfo getTypeInfoFromObjectInspector(ObjectInspector oi) { + // OPTIMIZATION for later. + // if (oi instanceof TypeInfoBasedObjectInspector) { + // TypeInfoBasedObjectInspector typeInfoBasedObjectInspector = + // (ObjectInspector)oi; + // return typeInfoBasedObjectInspector.getTypeInfo(); + // } + if (oi == null) { + return null; + } + + // Recursively going into ObjectInspector structure + TypeInfo result = null; + switch (oi.getCategory()) { + case PRIMITIVE: { + PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; + result = TypeInfoFactory.getPrimitiveTypeInfo(poi.getTypeName()); + break; + } + case LIST: { + ListObjectInspector loi = (ListObjectInspector) oi; + result = TypeInfoFactory + .getListTypeInfo(getTypeInfoFromObjectInspector(loi + .getListElementObjectInspector())); + break; + } + case MAP: { + MapObjectInspector moi = (MapObjectInspector) oi; + result = TypeInfoFactory.getMapTypeInfo( + getTypeInfoFromObjectInspector(moi.getMapKeyObjectInspector()), + getTypeInfoFromObjectInspector(moi.getMapValueObjectInspector())); + break; + } + case STRUCT: { + StructObjectInspector soi = (StructObjectInspector) oi; + List fields = soi.getAllStructFieldRefs(); + List fieldNames = new ArrayList(fields.size()); + List fieldTypeInfos = new ArrayList(fields.size()); + for (StructField f : fields) { + fieldNames.add(f.getFieldName()); + fieldTypeInfos.add(getTypeInfoFromObjectInspector(f + .getFieldObjectInspector())); + } + result = TypeInfoFactory.getStructTypeInfo(fieldNames, fieldTypeInfos); + break; + } + case UNION: { + UnionObjectInspector uoi = (UnionObjectInspector) oi; + List objectTypeInfos = new ArrayList(); + for (ObjectInspector eoi : uoi.getObjectInspectors()) { + objectTypeInfos.add(getTypeInfoFromObjectInspector(eoi)); + } + result = TypeInfoFactory.getUnionTypeInfo(objectTypeInfos); + break; + } + default: { + throw new RuntimeException("Unknown ObjectInspector category!"); + } + } + return result; + } + + public static ArrayList getTypeInfosFromTypeString(String typeString) { + TypeInfoParser parser = new TypeInfoParser(typeString); + return parser.parseTypeInfos(); + } + + public static TypeInfo getTypeInfoFromTypeString(String typeString) { + TypeInfoParser parser = new TypeInfoParser(typeString); + return parser.parseTypeInfos().get(0); + } +} diff --git serde/src/test/org/apache/hadoop/hive/serde2/thrift_test/CreateSequenceFile.java serde/src/test/org/apache/hadoop/hive/serde2/thrift_test/CreateSequenceFile.java index 8aef773..1fb49e5 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/thrift_test/CreateSequenceFile.java +++ serde/src/test/org/apache/hadoop/hive/serde2/thrift_test/CreateSequenceFile.java @@ -1,145 +1,145 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.thrift_test; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Random; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.serde2.ByteStream; -import org.apache.hadoop.hive.serde2.thrift.test.Complex; -import org.apache.hadoop.hive.serde2.thrift.test.IntString; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.mapred.JobConf; -import org.apache.thrift.TBase; -import org.apache.thrift.TException; -import org.apache.thrift.protocol.TBinaryProtocol; -import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.protocol.TProtocolFactory; -import org.apache.thrift.transport.TIOStreamTransport; - -/** - * CreateSequenceFile. - * - */ -public final class CreateSequenceFile { - - private CreateSequenceFile() { - // prevent instantiation - } - - public static void usage() { - System.out.println("Usage: CreateSequenceFile "); - System.exit(1); - } - - /** - * ThriftSerializer. - * - */ - public static class ThriftSerializer { - - private ByteStream.Output bos; - private TProtocol outProtocol; - - public ThriftSerializer() { - bos = new ByteStream.Output(); - TIOStreamTransport outTransport = new TIOStreamTransport(bos); - TProtocolFactory outFactory = new TBinaryProtocol.Factory(); - outProtocol = outFactory.getProtocol(outTransport); - } - - private BytesWritable bw = new BytesWritable(); - - public BytesWritable serialize(TBase base) throws TException { - bos.reset(); - base.write(outProtocol); - bw.set(bos.getData(), 0, bos.getCount()); - return bw; - } - } - - public static void main(String[] args) throws Exception { - - // Read parameters - int lines = 10; - List extraArgs = new ArrayList(); - for (int ai = 0; ai < args.length; ai++) { - if (args[ai].equals("-line") && ai + 1 < args.length) { - lines = Integer.parseInt(args[ai + 1]); - ai++; - } else { - extraArgs.add(args[ai]); - } - } - if (extraArgs.size() != 1) { - usage(); - } - - JobConf conf = new JobConf(CreateSequenceFile.class); - - ThriftSerializer serializer = new ThriftSerializer(); - - // Open files - SequenceFile.Writer writer = new SequenceFile.Writer(FileSystem.get(conf), - conf, new Path(extraArgs.get(0)), BytesWritable.class, - BytesWritable.class); - - // write to file - BytesWritable key = new BytesWritable(); - - Random rand = new Random(20081215); - - for (int i = 0; i < lines; i++) { - - ArrayList alist = new ArrayList(); - alist.add(i); - alist.add(i * 2); - alist.add(i * 3); - ArrayList slist = new ArrayList(); - slist.add("" + i * 10); - slist.add("" + i * 100); - slist.add("" + i * 1000); - ArrayList islist = new ArrayList(); - islist.add(new IntString(i * i, "" + i * i * i, i)); - HashMap hash = new HashMap(); - hash.put("key_" + i, "value_" + i); - - Complex complex = new Complex(rand.nextInt(), "record_" - + (new Integer(i)).toString(), alist, slist, islist, hash); - - Writable value = serializer.serialize(complex); - writer.append(key, value); - } - - // Add an all-null record - Complex complex = new Complex(0, null, null, null, null, null); - Writable value = serializer.serialize(complex); - writer.append(key, value); - - // Close files - writer.close(); - } - -} +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.thrift_test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Random; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.serde2.ByteStream; +import org.apache.hadoop.hive.serde2.thrift.test.Complex; +import org.apache.hadoop.hive.serde2.thrift.test.IntString; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapred.JobConf; +import org.apache.thrift.TBase; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.transport.TIOStreamTransport; + +/** + * CreateSequenceFile. + * + */ +public final class CreateSequenceFile { + + private CreateSequenceFile() { + // prevent instantiation + } + + public static void usage() { + System.out.println("Usage: CreateSequenceFile "); + System.exit(1); + } + + /** + * ThriftSerializer. + * + */ + public static class ThriftSerializer { + + private ByteStream.Output bos; + private TProtocol outProtocol; + + public ThriftSerializer() { + bos = new ByteStream.Output(); + TIOStreamTransport outTransport = new TIOStreamTransport(bos); + TProtocolFactory outFactory = new TBinaryProtocol.Factory(); + outProtocol = outFactory.getProtocol(outTransport); + } + + private BytesWritable bw = new BytesWritable(); + + public BytesWritable serialize(TBase base) throws TException { + bos.reset(); + base.write(outProtocol); + bw.set(bos.getData(), 0, bos.getCount()); + return bw; + } + } + + public static void main(String[] args) throws Exception { + + // Read parameters + int lines = 10; + List extraArgs = new ArrayList(); + for (int ai = 0; ai < args.length; ai++) { + if (args[ai].equals("-line") && ai + 1 < args.length) { + lines = Integer.parseInt(args[ai + 1]); + ai++; + } else { + extraArgs.add(args[ai]); + } + } + if (extraArgs.size() != 1) { + usage(); + } + + JobConf conf = new JobConf(CreateSequenceFile.class); + + ThriftSerializer serializer = new ThriftSerializer(); + + // Open files + SequenceFile.Writer writer = new SequenceFile.Writer(FileSystem.get(conf), + conf, new Path(extraArgs.get(0)), BytesWritable.class, + BytesWritable.class); + + // write to file + BytesWritable key = new BytesWritable(); + + Random rand = new Random(20081215); + + for (int i = 0; i < lines; i++) { + + ArrayList alist = new ArrayList(); + alist.add(i); + alist.add(i * 2); + alist.add(i * 3); + ArrayList slist = new ArrayList(); + slist.add("" + i * 10); + slist.add("" + i * 100); + slist.add("" + i * 1000); + ArrayList islist = new ArrayList(); + islist.add(new IntString(i * i, "" + i * i * i, i)); + HashMap hash = new HashMap(); + hash.put("key_" + i, "value_" + i); + + Complex complex = new Complex(rand.nextInt(), "record_" + + (new Integer(i)).toString(), alist, slist, islist, hash); + + Writable value = serializer.serialize(complex); + writer.append(key, value); + } + + // Add an all-null record + Complex complex = new Complex(0, null, null, null, null, null); + Writable value = serializer.serialize(complex); + writer.append(key, value); + + // Close files + writer.close(); + } + +}