diff --git hbase-handler/pom.xml hbase-handler/pom.xml index 7c3524c..9aca819 100644 --- hbase-handler/pom.xml +++ hbase-handler/pom.xml @@ -222,6 +222,19 @@ ${basedir}/src/java ${basedir}/src/test + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + diff --git hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseCompositeKey.java hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseCompositeKey.java index 5008f15..59afa8d 100644 --- hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseCompositeKey.java +++ hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseCompositeKey.java @@ -18,18 +18,24 @@ package org.apache.hadoop.hive.hbase; +import java.lang.reflect.Constructor; import java.util.ArrayList; import java.util.List; import java.util.Properties; +import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; import org.apache.hadoop.hive.serde2.lazy.LazyFactory; import org.apache.hadoop.hive.serde2.lazy.LazyObject; +import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase; +import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters; import org.apache.hadoop.hive.serde2.lazy.LazyStruct; import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; /** * HBaseCompositeKey extension of LazyStruct. All complex composite keys should extend this class @@ -96,4 +102,55 @@ public HBaseCompositeKey(LazySimpleStructObjectInspector oi) { return lazyObject; } + + /** + * HBaseKeyFactory implementation providing HBaseCompositeKey as a row key + */ + public static class HBaseCompositeKeyFactory implements HBaseKeyFactory, Configurable { + + private final Constructor constructor; + + private Configuration conf; + private Properties properties; + private SerDeParameters parameters; + + public HBaseCompositeKeyFactory(Class keyClass) throws Exception { + // see javadoc of HBaseCompositeKey + constructor = keyClass.getDeclaredConstructor( + LazySimpleStructObjectInspector.class, Properties.class, Configuration.class); + } + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + } + + @Override + public Configuration getConf() { + return conf; + } + + @Override + public void init(SerDeParameters parameters, Properties properties) { + this.parameters = parameters; + this.properties = properties; + } + + @Override + public LazyObjectBase createObject(ObjectInspector inspector) throws SerDeException { + try { + return (LazyObjectBase) constructor.newInstance(inspector, properties, conf); + } catch (Exception e) { + throw new SerDeException(e); + } + } + + @Override + public ObjectInspector createObjectInspector(TypeInfo type) + throws SerDeException { + return LazyFactory.createLazyObjectInspector(type, + parameters.getSeparators(), 1, parameters.getNullSequence(), + parameters.isEscaped(), parameters.getEscapeChar()); + } + } } diff --git hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseKeyFactory.java hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseKeyFactory.java new file mode 100644 index 0000000..80834eb --- /dev/null +++ hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseKeyFactory.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.hbase; + +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase; +import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; + +import java.util.Properties; + +/** + * Provides custom implementation of object and object inspector for hbase key. + * key object should implement LazyObjectBase + */ +public interface HBaseKeyFactory { + + /** + * initialize factory with properties + */ + void init(SerDeParameters parameters, Properties properties) throws SerDeException; + + /** + * create custom object inspector for hbase key + * @param type type information + */ + ObjectInspector createObjectInspector(TypeInfo type) throws SerDeException; + + /** + * create custom object for hbase key + * @param inspector OI create by {@link HBaseKeyFactory#createObjectInspector} + */ + LazyObjectBase createObject(ObjectInspector inspector) throws SerDeException; +} diff --git hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseLazyObjectFactory.java hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseLazyObjectFactory.java new file mode 100644 index 0000000..d466ff6 --- /dev/null +++ hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseLazyObjectFactory.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.hbase; + +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.lazy.LazyFactory; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.io.Text; + +import java.util.ArrayList; +import java.util.List; + +// Does same thing with LazyFactory#createLazyObjectInspector except that this replaces +// original keyOI with OI which is create by HBaseKeyFactory provided by serde property for hbase +public class HBaseLazyObjectFactory { + + public static ObjectInspector createLazyHBaseStructInspector( + List columnNames, List typeInfos, byte[] separators, + Text nullSequence, boolean lastColumnTakesRest, boolean escaped, + byte escapeChar, int index, HBaseKeyFactory factory) throws SerDeException { + ArrayList columnObjectInspectors = new ArrayList( + typeInfos.size()); + for (int i = 0; i < typeInfos.size(); i++) { + if (i == index && factory != null) { + columnObjectInspectors.add(factory.createObjectInspector(typeInfos.get(i))); + } else { + columnObjectInspectors.add(LazyFactory.createLazyObjectInspector( + typeInfos.get(i), separators, 1, nullSequence, escaped, escapeChar)); + } + } + return LazyObjectInspectorFactory.getLazySimpleStructObjectInspector( + columnNames, columnObjectInspectors, separators[0], nullSequence, + lastColumnTakesRest, escaped, escapeChar); + } +} diff --git hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java index 2cd65cb..c4e6f07 100644 --- hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java +++ hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.hbase; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -32,7 +31,6 @@ import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.SerDeUtils; -import org.apache.hadoop.hive.serde2.lazy.LazyFactory; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters; import org.apache.hadoop.hive.serde2.lazy.LazyUtils; @@ -54,6 +52,7 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.Writable; +import org.apache.hadoop.util.ReflectionUtils; /** * HBaseSerDe can be used to serialize object into an HBase table and @@ -67,6 +66,7 @@ public static final String HBASE_KEY_COL = ":key"; public static final String HBASE_PUT_TIMESTAMP = "hbase.put.timestamp"; public static final String HBASE_COMPOSITE_KEY_CLASS = "hbase.composite.key.class"; + public static final String HBASE_COMPOSITE_KEY_FACTORY = "hbase.composite.key.factory"; public static final String HBASE_SCAN_CACHE = "hbase.scan.cache"; public static final String HBASE_SCAN_CACHEBLOCKS = "hbase.scan.cacheblock"; public static final String HBASE_SCAN_BATCH = "hbase.scan.batch"; @@ -87,8 +87,7 @@ private final ByteStream.Output serializeStream = new ByteStream.Output(); private int iKey; private long putTimestamp; - private Class compositeKeyClass; - private Object compositeKeyObj; + private HBaseKeyFactory keyFactory; // used for serializing a field private byte [] separators; // the separators array @@ -123,22 +122,19 @@ public void initialize(Configuration conf, Properties tbl) initHBaseSerDeParameters(conf, tbl, getClass().getName()); - cachedObjectInspector = LazyFactory.createLazyStructInspector( + cachedObjectInspector = HBaseLazyObjectFactory.createLazyHBaseStructInspector( serdeParams.getColumnNames(), serdeParams.getColumnTypes(), serdeParams.getSeparators(), serdeParams.getNullSequence(), serdeParams.isLastColumnTakesRest(), serdeParams.isEscaped(), - serdeParams.getEscapeChar()); + serdeParams.getEscapeChar(), + iKey, keyFactory + ); cachedHBaseRow = new LazyHBaseRow( - (LazySimpleStructObjectInspector) cachedObjectInspector); - - if (compositeKeyClass != null) { - // initialize the constructor of the composite key class with its object inspector - initCompositeKeyClass(conf,tbl); - } + (LazySimpleStructObjectInspector) cachedObjectInspector, iKey, keyFactory); if (LOG.isDebugEnabled()) { LOG.debug("HBaseSerDe initialized with : columnNames = " @@ -469,16 +465,6 @@ private void initHBaseSerDeParameters( doColumnRegexMatching = Boolean.valueOf(tbl.getProperty(HBASE_COLUMNS_REGEX_MATCHING, "true")); - String compKeyClass = tbl.getProperty(HBaseSerDe.HBASE_COMPOSITE_KEY_CLASS); - - if (compKeyClass != null) { - try { - compositeKeyClass = job.getClassByName(compKeyClass); - } catch (ClassNotFoundException e) { - throw new SerDeException(e); - } - } - // Parse and initialize the HBase columns mapping columnsMapping = parseColumnsMapping(hbaseColumnsMapping, doColumnRegexMatching); @@ -547,6 +533,29 @@ private void initHBaseSerDeParameters( String hbaseTableStorageType = tbl.getProperty(HBaseSerDe.HBASE_TABLE_DEFAULT_STORAGE_TYPE); parseColumnStorageTypes(hbaseTableStorageType); setKeyColumnOffset(); + + try { + keyFactory = createKeyFactory(job, tbl); + if (keyFactory != null) { + keyFactory.init(serdeParams, tbl); + } + } catch (Exception e) { + throw new SerDeException(e); + } + } + + private HBaseKeyFactory createKeyFactory(Configuration job, Properties tbl) throws Exception { + String factoryClassName = tbl.getProperty(HBaseSerDe.HBASE_COMPOSITE_KEY_FACTORY); + if (factoryClassName != null) { + Class factoryClazz = Class.forName(factoryClassName); + return (HBaseKeyFactory) ReflectionUtils.newInstance(factoryClazz, job); + } + String keyClassName = tbl.getProperty(HBaseSerDe.HBASE_COMPOSITE_KEY_CLASS); + if (keyClassName != null) { + Class keyClass = Class.forName(keyClassName); + return new HBaseCompositeKey.HBaseCompositeKeyFactory(keyClass); + } + return null; } /** @@ -562,7 +571,7 @@ public Object deserialize(Writable result) throws SerDeException { throw new SerDeException(getClass().getName() + ": expects ResultWritable!"); } - cachedHBaseRow.init(((ResultWritable) result).getResult(), columnsMapping, compositeKeyObj); + cachedHBaseRow.init(((ResultWritable) result).getResult(), columnsMapping); return cachedHBaseRow; } @@ -692,8 +701,11 @@ public Writable serialize(Object obj, ObjectInspector objInspector) // the data to JSON string. Otherwise serialize the data in the // delimited way. serializeStream.reset(); + boolean isNotNull; - if (!foi.getCategory().equals(Category.PRIMITIVE) + if (colMap.hbaseRowKey && keyFactory instanceof HBaseWritableKeyFactory) { + isNotNull = ((HBaseWritableKeyFactory) keyFactory).serialize(f, foi, serializeStream); + } else if (!foi.getCategory().equals(Category.PRIMITIVE) && (declaredFields == null || declaredFields.get(i).getFieldObjectInspector().getCategory() .equals(Category.PRIMITIVE) || useJSONSerialize)) { @@ -828,48 +840,6 @@ private boolean serialize( } /** - * Initialize the composite key class with the objectinspector for the key - * - * @throws SerDeException - * */ - private void initCompositeKeyClass(Configuration conf,Properties tbl) throws SerDeException { - - int i = 0; - - // find the hbase row key - for (ColumnMapping colMap : columnsMapping) { - if (colMap.hbaseRowKey) { - break; - } - i++; - } - - ObjectInspector keyObjectInspector = ((LazySimpleStructObjectInspector) cachedObjectInspector) - .getAllStructFieldRefs().get(i).getFieldObjectInspector(); - - try { - compositeKeyObj = compositeKeyClass.getDeclaredConstructor( - LazySimpleStructObjectInspector.class, Properties.class, Configuration.class) - .newInstance( - ((LazySimpleStructObjectInspector) keyObjectInspector), tbl, conf); - } catch (IllegalArgumentException e) { - throw new SerDeException(e); - } catch (SecurityException e) { - throw new SerDeException(e); - } catch (InstantiationException e) { - throw new SerDeException(e); - } catch (IllegalAccessException e) { - throw new SerDeException(e); - } catch (InvocationTargetException e) { - throw new SerDeException(e); - } catch (NoSuchMethodException e) { - // the constructor wasn't defined in the implementation class. Flag error - throw new SerDeException("Constructor not defined in composite key class [" - + compositeKeyClass.getName() + "]", e); - } - } - - /** * @return the useJSONSerialize */ public boolean isUseJSONSerialize() { diff --git hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java index 8cd594b..bf3b6e7 100644 --- hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java +++ hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java @@ -34,8 +34,9 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.mapred.TableInputFormatBase; import org.apache.hadoop.hbase.mapred.TableOutputFormat; -import org.apache.hadoop.hbase.mapred.TableMapReduceUtil; +import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hive.hbase.HBaseSerDe.ColumnMapping; @@ -376,10 +377,19 @@ private void addHBaseDelegationToken(Configuration conf) throws IOException { @Override public void configureJobConf(TableDesc tableDesc, JobConf jobConf) { try { - TableMapReduceUtil.addDependencyJars(jobConf); - org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJars(jobConf, - HBaseStorageHandler.class, org.apache.hadoop.hbase.HBaseConfiguration.class); - } catch (IOException e) { + TableMapReduceUtil.addDependencyJars(jobConf, + HBaseStorageHandler.class, TableInputFormatBase.class, + // explicitly pull a class from each module + org.apache.hadoop.hbase.HConstants.class, // hbase-common + org.apache.hadoop.hbase.protobuf.generated.ClientProtos.class, // hbase-protocol + org.apache.hadoop.hbase.client.Put.class, // hbase-client + org.apache.hadoop.hbase.CompatibilityFactory.class, // hbase-hadoop-compat + // pull necessary dependencies + org.apache.zookeeper.ZooKeeper.class, + org.jboss.netty.channel.ChannelFactory.class, + com.google.protobuf.Message.class, + com.google.common.collect.Lists.class, + org.cloudera.htrace.Trace.class); } catch (IOException e) { throw new RuntimeException(e); } } diff --git hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseWritableKeyFactory.java hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseWritableKeyFactory.java new file mode 100644 index 0000000..7954a92 --- /dev/null +++ hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseWritableKeyFactory.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.hbase; + +import org.apache.hadoop.hive.serde2.ByteStream; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; + +import java.io.IOException; + +public interface HBaseWritableKeyFactory extends HBaseKeyFactory { + + /** + * serialize hive object in internal format of custom key + * + * @param object + * @param inspector + * @param output + * + * @return true if it's not null + * @throws IOException + */ + boolean serialize(Object object, ObjectInspector inspector, ByteStream.Output output) + throws IOException; + +} diff --git hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java index fc40195..27b063e 100644 --- hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java +++ hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java @@ -24,15 +24,14 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hive.hbase.HBaseSerDe.ColumnMapping; +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; import org.apache.hadoop.hive.serde2.lazy.LazyFactory; -import org.apache.hadoop.hive.serde2.lazy.LazyObject; +import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase; import org.apache.hadoop.hive.serde2.lazy.LazyStruct; import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; /** * LazyObject for storing an HBase row. The field of an HBase row can be @@ -45,72 +44,47 @@ */ private Result result; private List columnsMapping; - private Object compositeKeyObj; private ArrayList cachedList; - /** - * Construct a LazyHBaseRow object with the ObjectInspector. - */ + private final int iKey; + private final HBaseKeyFactory keyFactory; + public LazyHBaseRow(LazySimpleStructObjectInspector oi) { - super(oi); + this(oi, -1, null); } /** - * Set the HBase row data(a Result writable) for this LazyStruct. - * @see LazyHBaseRow#init(Result) + * Construct a LazyHBaseRow object with the ObjectInspector. */ - public void init(Result r, List columnsMapping) { - init(r, columnsMapping, null); + public LazyHBaseRow(LazySimpleStructObjectInspector oi, + int iKey, HBaseKeyFactory keyFactory) { + super(oi); + this.iKey = iKey; + this.keyFactory = keyFactory; } /** * Set the HBase row data(a Result writable) for this LazyStruct. - * * @see LazyHBaseRow#init(Result) - * - * @param compositeKeyClass - * custom implementation to interpret the composite key */ - public void init(Result r, List columnsMapping, Object compositeKeyObj) { - - result = r; + public void init(Result r, List columnsMapping) { + this.result = r; this.columnsMapping = columnsMapping; - this.compositeKeyObj = compositeKeyObj; setParsed(false); } - /** - * Parse the Result and fill each field. - * @see LazyStruct#parse() - */ - private void parse() { - - if (getFields() == null) { - List fieldRefs = - ((StructObjectInspector)getInspector()).getAllStructFieldRefs(); - LazyObject [] fields = new LazyObject[fieldRefs.size()]; - - for (int i = 0; i < fields.length; i++) { - ColumnMapping colMap = columnsMapping.get(i); - - if (colMap.qualifierName == null && !colMap.hbaseRowKey) { - // a column family - fields[i] = new LazyHBaseCellMap( - (LazyMapObjectInspector) fieldRefs.get(i).getFieldObjectInspector()); - continue; - } - - fields[i] = LazyFactory.createLazyObject( - fieldRefs.get(i).getFieldObjectInspector(), - colMap.binaryStorage.get(0)); - } - - setFields(fields); - setFieldInited(new boolean[fields.length]); + @Override + protected LazyObjectBase createLazyField(int fieldID, StructField fieldRef) throws SerDeException { + if (keyFactory != null && fieldID == iKey) { + return keyFactory.createObject(fieldRef.getFieldObjectInspector()); } - - Arrays.fill(getFieldInited(), false); - setParsed(true); + ColumnMapping colMap = columnsMapping.get(fieldID); + if (colMap.qualifierName == null && !colMap.hbaseRowKey) { + // a column family + return new LazyHBaseCellMap((LazyMapObjectInspector) fieldRef.getFieldObjectInspector()); + } + return LazyFactory.createLazyObject(fieldRef.getFieldObjectInspector(), + colMap.binaryStorage.get(0)); } /** @@ -127,16 +101,17 @@ private void parse() { */ @Override public Object getField(int fieldID) { - if (!getParsed()) { - parse(); - } - - Object value = uncheckedGetField(fieldID); + initFields(); + return uncheckedGetField(fieldID); + } - if (columnsMapping.get(fieldID).hbaseRowKey && compositeKeyObj != null) { - return compositeKeyObj; - } else { - return value; + private void initFields() { + if (getFields() == null) { + initLazyFields(oi.getAllStructFieldRefs()); + } + if (!getParsed()) { + Arrays.fill(getFieldInited(), false); + setParsed(true); } } @@ -149,7 +124,7 @@ public Object getField(int fieldID) { */ private Object uncheckedGetField(int fieldID) { - LazyObject [] fields = getFields(); + LazyObjectBase[] fields = getFields(); boolean [] fieldsInited = getFieldInited(); if (!fieldsInited[fieldID]) { @@ -182,12 +157,6 @@ private Object uncheckedGetField(int fieldID) { if (ref != null) { fields[fieldID].init(ref, 0, ref.getData().length); - - // if it was a row key and we have been provided a custom composite key class, initialize it - // with the bytes for the row key - if (colMap.hbaseRowKey && compositeKeyObj != null) { - ((LazyStruct) compositeKeyObj).init(ref, 0, ref.getData().length); - } } } @@ -203,9 +172,7 @@ private Object uncheckedGetField(int fieldID) { */ @Override public ArrayList getFieldsAsList() { - if (!getParsed()) { - parse(); - } + initFields(); if (cachedList == null) { cachedList = new ArrayList(); } else { diff --git hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseKeyFactory.java hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseKeyFactory.java new file mode 100644 index 0000000..2014531 --- /dev/null +++ hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseKeyFactory.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.hbase; + +import org.apache.hadoop.hive.serde2.ByteStream; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.StructObjectBaseInspector; +import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; +import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase; +import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; + +public class TestHBaseKeyFactory implements HBaseWritableKeyFactory { + + private static final String DELIMITER_PATTERN = "\\$\\$"; + private static final byte[] DELIMITER_BINARY = "$$".getBytes(); + + @Override + public void init(LazySimpleSerDe.SerDeParameters parameters, Properties properties) throws SerDeException { + } + + @Override + public ObjectInspector createObjectInspector(TypeInfo type) { + return new SlashSeparatedOI((StructTypeInfo)type); + } + + @Override + public LazyObjectBase createObject(ObjectInspector inspector) throws SerDeException { + return new DoubleDollarSeparated(); + } + + + @Override + public boolean serialize(Object object, ObjectInspector inspector, ByteStream.Output output) throws IOException { + if (inspector.getCategory() != ObjectInspector.Category.STRUCT) { + throw new IllegalStateException("invalid type value " + inspector.getTypeName()); + } + int current = output.getCount(); + for (Object element : ((StructObjectInspector)inspector).getStructFieldsDataAsList(object)) { + if (output.getCount() > current) { + output.write(DELIMITER_BINARY); + } + output.write(String.valueOf(element).getBytes()); + } + return output.getCount() > current; + } + + private static class DoubleDollarSeparated implements LazyObjectBase { + + private Object[] fields; + + @Override + public void init(ByteArrayRef bytes, int start, int length) { + fields = new String(bytes.getData(), start, length).split(DELIMITER_PATTERN); + } + + @Override + public Object getObject() { + return this; + } + } + + private static class SlashSeparatedOI extends StructObjectBaseInspector { + + private int length; + + private SlashSeparatedOI(StructTypeInfo type) { + List names = type.getAllStructFieldNames(); + List ois = new ArrayList(); + for (int i = 0; i < names.size(); i++) { + ois.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector); + } + init(names, ois, null); + } + + @Override + public Object getStructFieldData(Object data, StructField fieldRef) { + return ((DoubleDollarSeparated)data).fields[((MyField)fieldRef).getFieldID()]; + } + + @Override + public List getStructFieldsDataAsList(Object data) { + return Arrays.asList(((DoubleDollarSeparated)data).fields); + } + } +} diff --git hbase-handler/src/test/queries/positive/hbase_custom_key.q hbase-handler/src/test/queries/positive/hbase_custom_key.q new file mode 100644 index 0000000..f66081f --- /dev/null +++ hbase-handler/src/test/queries/positive/hbase_custom_key.q @@ -0,0 +1,26 @@ +CREATE TABLE hbase_ck_1(key struct, value string) +STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +WITH SERDEPROPERTIES ( + "hbase.columns.mapping" = ":key,cf:string", + "hbase.composite.key.class"="org.apache.hadoop.hive.hbase.HBaseTestCompositeKey"); + +CREATE TABLE hbase_ck_2(key struct, value string) +STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +WITH SERDEPROPERTIES ( + "hbase.table.name" = "hbase_custom", + "hbase.columns.mapping" = ":key,cf:string", + "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.TestHBaseKeyFactory"); + +CREATE EXTERNAL TABLE hbase_ck_3(key string, value string) +STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +WITH SERDEPROPERTIES ( + "hbase.table.name" = "hbase_custom", + "hbase.columns.mapping" = ":key,cf:string"); + +from src tablesample (1 rows) +insert into table hbase_ck_1 select struct('1000','2000','3000'),'value' +insert into table hbase_ck_2 select struct('1000','2000','3000'),'value'; + +select * from hbase_ck_1; +select * from hbase_ck_2; +select * from hbase_ck_3; diff --git hbase-handler/src/test/results/positive/hbase_custom_key.q.out hbase-handler/src/test/results/positive/hbase_custom_key.q.out new file mode 100644 index 0000000..653f6b4 --- /dev/null +++ hbase-handler/src/test/results/positive/hbase_custom_key.q.out @@ -0,0 +1,88 @@ +PREHOOK: query: CREATE TABLE hbase_ck_1(key struct, value string) +STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +WITH SERDEPROPERTIES ( + "hbase.columns.mapping" = ":key,cf:string", + "hbase.composite.key.class"="org.apache.hadoop.hive.hbase.HBaseTestCompositeKey") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE hbase_ck_1(key struct, value string) +STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +WITH SERDEPROPERTIES ( + "hbase.columns.mapping" = ":key,cf:string", + "hbase.composite.key.class"="org.apache.hadoop.hive.hbase.HBaseTestCompositeKey") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@hbase_ck_1 +PREHOOK: query: CREATE TABLE hbase_ck_2(key struct, value string) +STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +WITH SERDEPROPERTIES ( + "hbase.table.name" = "hbase_custom", + "hbase.columns.mapping" = ":key,cf:string", + "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.TestHBaseKeyFactory") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE hbase_ck_2(key struct, value string) +STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +WITH SERDEPROPERTIES ( + "hbase.table.name" = "hbase_custom", + "hbase.columns.mapping" = ":key,cf:string", + "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.TestHBaseKeyFactory") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@hbase_ck_2 +PREHOOK: query: CREATE EXTERNAL TABLE hbase_ck_3(key string, value string) +STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +WITH SERDEPROPERTIES ( + "hbase.table.name" = "hbase_custom", + "hbase.columns.mapping" = ":key,cf:string") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE EXTERNAL TABLE hbase_ck_3(key string, value string) +STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +WITH SERDEPROPERTIES ( + "hbase.table.name" = "hbase_custom", + "hbase.columns.mapping" = ":key,cf:string") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@hbase_ck_3 +PREHOOK: query: from src tablesample (1 rows) +insert into table hbase_ck_1 select struct('1000','2000','3000'),'value' +insert into table hbase_ck_2 select struct('1000','2000','3000'),'value' +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@hbase_ck_1 +PREHOOK: Output: default@hbase_ck_2 +POSTHOOK: query: from src tablesample (1 rows) +insert into table hbase_ck_1 select struct('1000','2000','3000'),'value' +insert into table hbase_ck_2 select struct('1000','2000','3000'),'value' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@hbase_ck_1 +POSTHOOK: Output: default@hbase_ck_2 +PREHOOK: query: select * from hbase_ck_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@hbase_ck_1 +#### A masked pattern was here #### +POSTHOOK: query: select * from hbase_ck_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hbase_ck_1 +#### A masked pattern was here #### +{"col1":"1","col2":"0","col3":"0"} value +PREHOOK: query: select * from hbase_ck_2 +PREHOOK: type: QUERY +PREHOOK: Input: default@hbase_ck_2 +#### A masked pattern was here #### +POSTHOOK: query: select * from hbase_ck_2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hbase_ck_2 +#### A masked pattern was here #### +{"col1":"1000","col2":"2000","col3":"3000"} value +PREHOOK: query: select * from hbase_ck_3 +PREHOOK: type: QUERY +PREHOOK: Input: default@hbase_ck_3 +#### A masked pattern was here #### +POSTHOOK: query: select * from hbase_ck_3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hbase_ck_3 +#### A masked pattern was here #### +1000$$2000$$3000 value diff --git itests/util/pom.xml itests/util/pom.xml index 9885c53..695320e 100644 --- itests/util/pom.xml +++ itests/util/pom.xml @@ -46,6 +46,12 @@ org.apache.hive + hive-hbase-handler + ${project.version} + tests + + + org.apache.hive hive-metastore ${project.version} diff --git serde/src/java/org/apache/hadoop/hive/serde2/StructObject.java serde/src/java/org/apache/hadoop/hive/serde2/StructObject.java new file mode 100644 index 0000000..b7efff0 --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/StructObject.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2; + +import java.util.List; + +public interface StructObject { + + Object getField(int fieldID); + + List getFieldsAsList(); +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/StructObjectBaseInspector.java serde/src/java/org/apache/hadoop/hive/serde2/StructObjectBaseInspector.java new file mode 100644 index 0000000..9a1a3b2 --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/StructObjectBaseInspector.java @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2; + +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; + +import java.util.ArrayList; +import java.util.List; + +public abstract class StructObjectBaseInspector extends StructObjectInspector { + + protected static class MyField implements StructField { + + protected final int fieldID; + protected final String fieldName; + protected final String fieldComment; + protected final ObjectInspector fieldObjectInspector; + + public MyField(int fieldID, String fieldName, + ObjectInspector fieldObjectInspector, String fieldComment) { + this.fieldID = fieldID; + this.fieldName = fieldName.toLowerCase(); + this.fieldObjectInspector = fieldObjectInspector; + this.fieldComment = fieldComment; + } + + public int getFieldID() { + return fieldID; + } + + public String getFieldName() { + return fieldName; + } + + public ObjectInspector getFieldObjectInspector() { + return fieldObjectInspector; + } + + public String getFieldComment() { + return fieldComment; + } + @Override + public String toString() { + return "" + fieldID + ":" + fieldName; + } + } + + private List fields; + + protected StructObjectBaseInspector() { + super(); + } + /** + * Call ObjectInspectorFactory.getLazySimpleStructObjectInspector instead. + */ + public StructObjectBaseInspector(List structFieldNames, + List structFieldObjectInspectors) { + init(structFieldNames, structFieldObjectInspectors, null); + } + + public StructObjectBaseInspector(List structFieldNames, + List structFieldObjectInspectors, + List structFieldComments) { + init(structFieldNames, structFieldObjectInspectors, structFieldComments); + } + + protected void init(List structFieldNames, + List structFieldObjectInspectors, + List structFieldComments) { + assert (structFieldNames.size() == structFieldObjectInspectors.size()); + assert (structFieldComments == null || + (structFieldNames.size() == structFieldComments.size())); + + fields = new ArrayList(structFieldNames.size()); + for (int i = 0; i < structFieldNames.size(); i++) { + fields.add(createField(i, + structFieldNames.get(i), structFieldObjectInspectors.get(i), + structFieldComments == null ? null : structFieldComments.get(i))); + } + } + + protected MyField createField(int index, String fieldName, ObjectInspector fieldOI, String comment) { + return new MyField(index, fieldName, fieldOI, comment); + } + + @Override + public String getTypeName() { + return ObjectInspectorUtils.getStandardStructTypeName(this); + } + + @Override + public final Category getCategory() { + return Category.STRUCT; + } + + @Override + public StructField getStructFieldRef(String fieldName) { + return ObjectInspectorUtils.getStandardStructFieldRef(fieldName, fields); + } + + @Override + public List getAllStructFieldRefs() { + return fields; + } +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStructBase.java serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStructBase.java index 1fd6853..fd06f58 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStructBase.java +++ serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStructBase.java @@ -23,13 +23,14 @@ import java.util.List; import org.apache.hadoop.hive.serde2.SerDeStatsStruct; +import org.apache.hadoop.hive.serde2.StructObject; import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -public abstract class ColumnarStructBase implements SerDeStatsStruct { +public abstract class ColumnarStructBase implements StructObject, SerDeStatsStruct { class FieldInfo { LazyObjectBase field; diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java index 10f4c05..9b5ccbe 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java @@ -25,7 +25,7 @@ * A LazyObject can represent any primitive object or hierarchical object like * array, map or struct. */ -public abstract class LazyObject extends LazyObjectBase { +public abstract class LazyObject implements LazyObjectBase { protected OI oi; diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObjectBase.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObjectBase.java index 3334dff..7e42b3f 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObjectBase.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObjectBase.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hive.serde2.lazy; -public abstract class LazyObjectBase { +public interface LazyObjectBase { /** * Set the data for this LazyObjectBase. We take ByteArrayRef instead of byte[] so @@ -33,12 +33,12 @@ * The length of the data, starting from "start" * @see ByteArrayRef */ - public abstract void init(ByteArrayRef bytes, int start, int length); + void init(ByteArrayRef bytes, int start, int length); /** * If the LazyObjectBase is a primitive Object, then deserialize it and return the * actual primitive Object. Otherwise (array, map, struct), return this. */ - public abstract Object getObject(); + Object getObject(); } diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java index 8a1ea46..a01cd66 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java @@ -23,10 +23,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.SerDeStatsStruct; +import org.apache.hadoop.hive.serde2.StructObject; import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.io.Text; /** @@ -36,8 +37,8 @@ * LazyStruct does not deal with the case of a NULL struct. That is handled by * the parent LazyObject. */ -public class LazyStruct extends LazyNonPrimitive implements - SerDeStatsStruct { +public class LazyStruct extends LazyNonPrimitive + implements StructObject, SerDeStatsStruct { private static Log LOG = LogFactory.getLog(LazyStruct.class.getName()); @@ -62,7 +63,7 @@ /** * The fields of the struct. */ - LazyObject[] fields; + LazyObjectBase[] fields; /** * Whether init() has been called on the field or not. */ @@ -101,17 +102,7 @@ private void parse() { byte escapeChar = oi.getEscapeChar(); if (fields == null) { - List fieldRefs = ((StructObjectInspector) oi) - .getAllStructFieldRefs(); - fields = new LazyObject[fieldRefs.size()]; - for (int i = 0; i < fields.length; i++) { - fields[i] = LazyFactory.createLazyObject(fieldRefs.get(i) - .getFieldObjectInspector()); - } - fieldInited = new boolean[fields.length]; - // Extra element to make sure we have the same formula to compute the - // length of each element of the array. - startPosition = new int[fields.length + 1]; + initLazyFields(oi.getAllStructFieldRefs()); } int structByteEnd = start + length; @@ -172,6 +163,25 @@ private void parse() { parsed = true; } + protected final void initLazyFields(List fieldRefs) { + fields = new LazyObjectBase[fieldRefs.size()]; + for (int i = 0; i < fields.length; i++) { + try { + fields[i] = createLazyField(i, fieldRefs.get(i)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + fieldInited = new boolean[fields.length]; + // Extra element to make sure we have the same formula to compute the + // length of each element of the array. + startPosition = new int[fields.length + 1]; + } + + protected LazyObjectBase createLazyField(int fieldID, StructField fieldRef) throws SerDeException { + return LazyFactory.createLazyObject(fieldRef.getFieldObjectInspector()); + } + /** * Get one field out of the struct. * @@ -221,14 +231,14 @@ private Object uncheckedGetField(int fieldID) { return fields[fieldID].getObject(); } - ArrayList cachedList; + List cachedList; /** * Get the values of the fields as an ArrayList. * * @return The values of the fields as an ArrayList. */ - public ArrayList getFieldsAsList() { + public List getFieldsAsList() { if (!parsed) { parse(); } @@ -256,7 +266,7 @@ protected void setParsed(boolean parsed) { this.parsed = parsed; } - protected LazyObject[] getFields() { + protected LazyObjectBase[] getFields() { return fields; } diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java index 8a5386a..0813700 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java @@ -23,7 +23,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hive.serde2.lazy.LazyStruct; +import org.apache.hadoop.hive.serde2.StructObject; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.StructField; @@ -184,7 +184,7 @@ public Object getStructFieldData(Object data, StructField fieldRef) { if (data == null) { return null; } - LazyStruct struct = (LazyStruct) data; + StructObject struct = (StructObject) data; MyField f = (MyField) fieldRef; int fieldID = f.getFieldID(); @@ -198,15 +198,8 @@ public Object getStructFieldData(Object data, StructField fieldRef) { if (data == null) { return null; } - - // Iterate over all the fields picking up the nested structs within them - List result = new ArrayList(fields.size()); - - for (MyField myField : fields) { - result.add(getStructFieldData(data, myField)); - } - - return result; + StructObject struct = (StructObject) data; + return struct.getFieldsAsList(); } // For LazyStruct diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryObject.java serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryObject.java index 598683f..b3625b3 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryObject.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryObject.java @@ -27,7 +27,7 @@ * A LazyBinaryObject can represent any primitive object or hierarchical object * like string, list, map or struct. */ -public abstract class LazyBinaryObject extends LazyObjectBase { +public abstract class LazyBinaryObject implements LazyObjectBase { OI oi; diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryStruct.java serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryStruct.java index caf3517..98a35c7 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryStruct.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryStruct.java @@ -24,6 +24,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.serde2.SerDeStatsStruct; +import org.apache.hadoop.hive.serde2.StructObject; import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils.RecordInfo; import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryStructObjectInspector; @@ -42,8 +43,8 @@ * Following B, there is another section A and B. This pattern repeats until the * all struct fields are serialized. */ -public class LazyBinaryStruct extends - LazyBinaryNonPrimitive implements SerDeStatsStruct { +public class LazyBinaryStruct extends LazyBinaryNonPrimitive + implements StructObject, SerDeStatsStruct { private static Log LOG = LogFactory.getLog(LazyBinaryStruct.class.getName());