diff --git common/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java common/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java new file mode 100644 index 0000000..b230410 --- /dev/null +++ common/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.type; + +import org.apache.commons.lang.StringUtils; + +public abstract class HiveBaseChar { + protected String value; + protected int characterLength = -1; + + protected HiveBaseChar() { + } + + /** + * Sets the string value to a new value, obeying the max length defined for this object. + * @param val new value + */ + public void setValue(String val, int maxLength) { + characterLength = -1; + value = HiveBaseChar.enforceMaxLength(val, maxLength); + } + + public void setValue(HiveBaseChar val, int maxLength) { + if ((maxLength < 0) + || (val.characterLength > 0 && val.characterLength <= maxLength)) { + // No length enforcement required, or source length is less than max length. + // We can copy the source value as-is. + value = val.value; + this.characterLength = val.characterLength; + } else { + setValue(val.value, maxLength); + } + } + + public static String enforceMaxLength(String val, int maxLength) { + String value = val; + + if (maxLength > 0) { + int valLength = val.codePointCount(0, val.length()); + if (valLength > maxLength) { + // Truncate the excess trailing spaces to fit the character length. + // Also make sure we take supplementary chars into account. + value = val.substring(0, val.offsetByCodePoints(0, maxLength)); + } + } + return value; + } + + public String getValue() { + return value; + } + + public int getCharacterLength() { + if (characterLength < 0) { + characterLength = value.codePointCount(0, value.length()); + } + return characterLength; + } +} diff --git common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java new file mode 100644 index 0000000..1a3c2fc --- /dev/null +++ common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.type; + +/** + * + * HiveVarChar. + * String wrapper to support SQL VARCHAR features. + * Max string length is enforced. + * + */ +public class HiveVarchar extends HiveBaseChar + implements Comparable { + + public static final int MAX_VARCHAR_LENGTH = Integer.MAX_VALUE; + + public HiveVarchar() { + } + + public HiveVarchar(String val, int len) { + setValue(val, len); + } + + public HiveVarchar(HiveVarchar hc, int len) { + setValue(hc, len); + } + + /** + * Set the new value + */ + public void setValue(String val) { + super.setValue(val, -1); + } + + public void setValue(HiveVarchar hc) { + super.setValue(hc.getValue(), -1); + } + + @Override + public String toString() { + return getValue(); + } + + public int compareTo(HiveVarchar rhs) { + if (rhs == this) { + return 0; + } + return this.getValue().compareTo(rhs.getValue()); + } + + public boolean equals(HiveVarchar rhs) { + if (rhs == this) { + return true; + } + return this.getValue().equals(rhs.getValue()); + } + + public int hashCode() { + return getValue().hashCode(); + } +} diff --git common/src/test/org/apache/hadoop/hive/common/type/TestHiveVarchar.java common/src/test/org/apache/hadoop/hive/common/type/TestHiveVarchar.java new file mode 100644 index 0000000..6f9b0bb --- /dev/null +++ common/src/test/org/apache/hadoop/hive/common/type/TestHiveVarchar.java @@ -0,0 +1,143 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.type; + +import junit.framework.TestCase; + +import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.common.LogUtils; +import org.apache.hadoop.hive.common.LogUtils.LogInitializationException; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Random; + + +public class TestHiveVarchar extends TestCase { + public TestHiveVarchar() { + super(); + } + + static Random rnd = new Random(); + + public static int getRandomSupplementaryChar() { + int lowSurrogate = 0xDC00 + rnd.nextInt(1024); + //return 0xD8000000 + lowSurrogate; + int highSurrogate = 0xD800; + return Character.toCodePoint((char)highSurrogate, (char)lowSurrogate); + } + + public static int getRandomCodePoint() { + int codePoint; + if (rnd.nextDouble() < 0.50) { + codePoint = 32 + rnd.nextInt(90); + } else { + codePoint = getRandomSupplementaryChar(); + } + if (!Character.isValidCodePoint(codePoint)) { + System.out.println(Integer.toHexString(codePoint) + " is not a valid code point"); + } + return codePoint; + } + + public static int getRandomCodePoint(int excludeChar) { + while (true) { + int codePoint = getRandomCodePoint(); + if (codePoint != excludeChar) { + return codePoint; + } + } + } + + public void testStringLength() throws Exception { + int strLen = 20; + int[] lengths = { 15, 20, 25 }; + // Try with supplementary characters + for (int idx1 = 0; idx1 < lengths.length; ++idx1) { + // Create random test string + StringBuffer sb = new StringBuffer(); + int curLen = lengths[idx1]; + for (int idx2 = 0; idx2 < curLen; ++idx2) { + sb.appendCodePoint(getRandomCodePoint(' ')); + } + String testString = sb.toString(); + assertEquals(curLen, testString.codePointCount(0, testString.length())); + String enforcedString = HiveBaseChar.enforceMaxLength(testString, strLen); + if (curLen <= strLen) { + // No truncation needed + assertEquals(testString, enforcedString); + } else { + // String should have been truncated. + assertEquals(strLen, enforcedString.codePointCount(0, enforcedString.length())); + } + } + + // Try with ascii chars + String[] testStrings = { + "abcdefg", + "abcdefghijklmnopqrst", + "abcdefghijklmnopqrstuvwxyz" + }; + for (String testString : testStrings) { + int curLen = testString.length(); + assertEquals(curLen, testString.codePointCount(0, testString.length())); + String enforcedString = HiveBaseChar.enforceMaxLength(testString, strLen); + if (curLen <= strLen) { + // No truncation needed + assertEquals(testString, enforcedString); + } else { + // String should have been truncated. + assertEquals(strLen, enforcedString.codePointCount(0, enforcedString.length())); + } + } + } + + public void testComparison() throws Exception { + HiveVarchar hc1 = new HiveVarchar("abcd", 20); + HiveVarchar hc2 = new HiveVarchar("abcd", 20); + + // Identical strings should be equal + assertTrue(hc1.equals(hc2)); + assertTrue(hc2.equals(hc1)); + assertEquals(0, hc1.compareTo(hc2)); + assertEquals(0, hc2.compareTo(hc1)); + + // Unequal strings + hc2 = new HiveVarchar("abcde", 20); + assertFalse(hc1.equals(hc2)); + assertFalse(hc2.equals(hc1)); + assertFalse(0 == hc1.compareTo(hc2)); + assertFalse(0 == hc2.compareTo(hc1)); + + // Trailing spaces are significant + hc2 = new HiveVarchar("abcd ", 30); + + assertFalse(hc1.equals(hc2)); + assertFalse(hc2.equals(hc1)); + assertFalse(0 == hc1.compareTo(hc2)); + assertFalse(0 == hc2.compareTo(hc1)); + + // Leading spaces are significant + hc2 = new HiveVarchar(" abcd", 20); + assertFalse(hc1.equals(hc2)); + assertFalse(hc2.equals(hc1)); + assertFalse(0 == hc1.compareTo(hc2)); + assertFalse(0 == hc2.compareTo(hc1)); + } +} diff --git contrib/src/java/org/apache/hadoop/hive/contrib/util/typedbytes/TypedBytesRecordReader.java contrib/src/java/org/apache/hadoop/hive/contrib/util/typedbytes/TypedBytesRecordReader.java index 8bffa02..e0a5b99 100644 --- contrib/src/java/org/apache/hadoop/hive/contrib/util/typedbytes/TypedBytesRecordReader.java +++ contrib/src/java/org/apache/hadoop/hive/contrib/util/typedbytes/TypedBytesRecordReader.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; +import org.apache.hadoop.hive.serde2.typeinfo.ParameterizedPrimitiveTypeUtils; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.FloatWritable; @@ -90,8 +91,9 @@ public void initialize(InputStream in, Configuration conf, Properties tbl) throw for (String columnType : columnTypes) { PrimitiveTypeEntry dstTypeEntry = PrimitiveObjectInspectorUtils .getTypeEntryFromTypeName(columnType); - dstOIns.add(PrimitiveObjectInspectorFactory - .getPrimitiveWritableObjectInspector(dstTypeEntry.primitiveCategory)); + dstOIns.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( + dstTypeEntry.primitiveCategory, + ParameterizedPrimitiveTypeUtils.getTypeParamsFromPrimitiveTypeEntry(dstTypeEntry))); } } @@ -154,8 +156,9 @@ public int next(Writable data) throws IOException { PrimitiveTypeEntry srcTypeEntry = PrimitiveObjectInspectorUtils .getTypeEntryFromTypeName(typeName); srcOIns - .add(PrimitiveObjectInspectorFactory - .getPrimitiveWritableObjectInspector(srcTypeEntry.primitiveCategory)); + .add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( + srcTypeEntry.primitiveCategory, + ParameterizedPrimitiveTypeUtils.getTypeParamsFromPrimitiveTypeEntry(srcTypeEntry))); converters.add(ObjectInspectorConverters.getConverter(srcOIns.get(pos), dstOIns.get(pos))); } else { diff --git data/files/datatypes.txt data/files/datatypes.txt index aabdb6e..10daa1b 100644 --- data/files/datatypes.txt +++ data/files/datatypes.txt @@ -1,3 +1,3 @@ -\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N --1false-1.1\N\N\N-1-1-1.0-1\N\N\N\N\N\N -1true1.11121x2ykva92.2111.01abcd1111213142212212x1abcd22012-04-22 09:00:00.123456789123456789.0123456YWJjZA==2013-01-01 +\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N +-1false-1.1\N\N\N-1-1-1.0-1\N\N\N\N\N\N\N +1true1.11121x2ykva92.2111.01abcd1111213142212212x1abcd22012-04-22 09:00:00.123456789123456789.0123456YWJjZA==2013-01-01abc123 diff --git data/files/vc1.txt data/files/vc1.txt new file mode 100644 index 0000000..d0f9952 --- /dev/null +++ data/files/vc1.txt @@ -0,0 +1,3 @@ +1abc +2abc +3 abc diff --git jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveBaseResultSet.java jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveBaseResultSet.java index d18a6a3..552f097 100644 --- jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveBaseResultSet.java +++ jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveBaseResultSet.java @@ -54,6 +54,7 @@ protected List row; protected List columnNames; protected List columnTypes; + protected List columnAttributes; public boolean absolute(int row) throws SQLException { throw new SQLException("Method not supported"); @@ -342,7 +343,7 @@ public long getLong(String columnName) throws SQLException { } public ResultSetMetaData getMetaData() throws SQLException { - return new HiveResultSetMetaData(columnNames, columnTypes); + return new HiveResultSetMetaData(columnNames, columnTypes, columnAttributes); } public Reader getNCharacterStream(int arg0) throws SQLException { diff --git jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveMetaDataResultSet.java jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveMetaDataResultSet.java index 9f7ae42..1bb2c42 100644 --- jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveMetaDataResultSet.java +++ jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveMetaDataResultSet.java @@ -44,6 +44,11 @@ public HiveMetaDataResultSet(final List columnNames } else { this.columnTypes = new ArrayList(); } + // Safe to assume the types here do not require column attributes? + this.columnAttributes = new ArrayList(); + for (int idx = 0; idx < columnTypes.size(); ++idx) { + this.columnAttributes.add(null); + } } @Override diff --git jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveQueryResultSet.java jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveQueryResultSet.java index 3215178..4a501a8 100644 --- jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveQueryResultSet.java +++ jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveQueryResultSet.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hive.service.HiveInterface; import org.apache.hadoop.hive.service.HiveServerException; import org.apache.hadoop.io.BytesWritable; +import org.apache.hive.service.cli.Type; /** * HiveQueryResultSet. @@ -80,6 +81,7 @@ private void initSerde() throws SQLException { List schema = fullSchema.getFieldSchemas(); columnNames = new ArrayList(); columnTypes = new ArrayList(); + columnAttributes = new ArrayList(); StringBuilder namesSb = new StringBuilder(); StringBuilder typesSb = new StringBuilder(); @@ -90,9 +92,18 @@ private void initSerde() throws SQLException { typesSb.append(","); } columnNames.add(schema.get(pos).getName()); - columnTypes.add(schema.get(pos).getType()); + Type dataType = Type.getType(schema.get(pos).getType()); + if (dataType.isQualifiedType()) { + columnTypes.add(dataType.getName().toLowerCase()); + columnAttributes.add( + JdbcColumn.getColumnAttributesFromType(schema.get(pos).getType(), dataType)); + } else { + // types without qualifiers can just use the original type string. + columnTypes.add(schema.get(pos).getType()); + columnAttributes.add(null); + } namesSb.append(schema.get(pos).getName()); - typesSb.append(schema.get(pos).getType()); + typesSb.append(schema.get(pos).getType()); // SerDe needs to use the qualified type name } } String names = namesSb.toString(); diff --git jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveResultSetMetaData.java jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveResultSetMetaData.java index 94b6ecd..4838fc2 100644 --- jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveResultSetMetaData.java +++ jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveResultSetMetaData.java @@ -31,11 +31,14 @@ public class HiveResultSetMetaData implements java.sql.ResultSetMetaData { private final List columnNames; private final List columnTypes; + private final List columnAttributes; public HiveResultSetMetaData(List columnNames, - List columnTypes) { + List columnTypes, + List columnAttributes) { this.columnNames = columnNames; this.columnTypes = columnTypes; + this.columnAttributes = columnAttributes; } public String getCatalogName(int column) throws SQLException { @@ -53,7 +56,7 @@ public int getColumnCount() throws SQLException { public int getColumnDisplaySize(int column) throws SQLException { int columnType = getColumnType(column); - return JdbcColumn.columnDisplaySize(columnType); + return JdbcColumn.columnDisplaySize(columnType, columnAttributes.get(column - 1)); } public String getColumnLabel(int column) throws SQLException { @@ -96,6 +99,8 @@ public String getColumnTypeName(int column) throws SQLException { String type = columnTypes.get(column - 1); if ("string".equalsIgnoreCase(type)) { return serdeConstants.STRING_TYPE_NAME; + } if ("varchar".equalsIgnoreCase(type)) { + return serdeConstants.VARCHAR_TYPE_NAME; } else if ("float".equalsIgnoreCase(type)) { return serdeConstants.FLOAT_TYPE_NAME; } else if ("double".equalsIgnoreCase(type)) { @@ -130,13 +135,13 @@ public String getColumnTypeName(int column) throws SQLException { public int getPrecision(int column) throws SQLException { int columnType = getColumnType(column); - return JdbcColumn.columnPrecision(columnType); + return JdbcColumn.columnPrecision(columnType, columnAttributes.get(column - 1)); } public int getScale(int column) throws SQLException { int columnType = getColumnType(column); - return JdbcColumn.columnScale(columnType); + return JdbcColumn.columnScale(columnType, columnAttributes.get(column - 1)); } public String getSchemaName(int column) throws SQLException { diff --git jdbc/src/java/org/apache/hadoop/hive/jdbc/JdbcColumn.java jdbc/src/java/org/apache/hadoop/hive/jdbc/JdbcColumn.java index d3a87e2..24406b6 100644 --- jdbc/src/java/org/apache/hadoop/hive/jdbc/JdbcColumn.java +++ jdbc/src/java/org/apache/hadoop/hive/jdbc/JdbcColumn.java @@ -21,6 +21,13 @@ import java.sql.SQLException; import java.sql.Types; +import org.apache.hadoop.hive.serde2.typeinfo.BaseTypeParams; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hive.service.cli.Type; +import org.apache.hive.service.cli.TypeQualifiers; + /** * Column metadata. */ @@ -31,15 +38,53 @@ private final String type; private final String comment; private final int ordinalPos; + private ColumnAttributes columnAttributes; + + static class ColumnAttributes { + public int precision = 0; + public int scale = 0; + + public ColumnAttributes() { + } + + public ColumnAttributes(int precision, int scale) { + this.precision = precision; + this.scale = scale; + } + } JdbcColumn(String columnName, String tableName, String tableCatalog - , String type, String comment, int ordinalPos) { + , String typeName, String comment, int ordinalPos) { this.columnName = columnName; this.tableName = tableName; this.tableCatalog = tableCatalog; - this.type = type; this.comment = comment; this.ordinalPos = ordinalPos; + Type dataType = Type.getType(typeName); + if (dataType.isQualifiedType()) { + this.type = dataType.getName().toLowerCase(); + columnAttributes = getColumnAttributesFromType(typeName, dataType); + } else { + this.type = typeName; + } + } + + static ColumnAttributes getColumnAttributesFromType(String typeName, Type dataType) { + ColumnAttributes ret = null; + if (dataType.isQualifiedType()) { + // Parse the type string to see if we need to get type qualifiers + PrimitiveTypeInfo pti = (PrimitiveTypeInfo) TypeInfoFactory.getPrimitiveTypeInfo(typeName); + + BaseTypeParams typeParams = pti.getParameters(); + if (typeParams != null) { + ret = new ColumnAttributes(); + if (typeParams.hasCharacterMaximumLength()) { + ret.precision = typeParams.getCharacterMaximumLength(); + } + } + + } + return ret; } public String getColumnName() { @@ -62,22 +107,26 @@ public Integer getSqlType() throws SQLException { return Utils.hiveTypeToSqlType(type); } - static int columnDisplaySize(int columnType) throws SQLException { + static int columnDisplaySize(int columnType, ColumnAttributes columnAttributes) + throws SQLException { // according to hiveTypeToSqlType possible options are: switch(columnType) { case Types.BOOLEAN: - return columnPrecision(columnType); + return columnPrecision(columnType, columnAttributes); case Types.VARCHAR: + if (columnAttributes != null) { + return columnAttributes.precision; + } return Integer.MAX_VALUE; // hive has no max limit for strings case Types.TINYINT: case Types.SMALLINT: case Types.INTEGER: case Types.BIGINT: - return columnPrecision(columnType) + 1; // allow +/- + return columnPrecision(columnType, columnAttributes) + 1; // allow +/- case Types.DATE: return 10; case Types.TIMESTAMP: - return columnPrecision(columnType); + return columnPrecision(columnType, columnAttributes); // see http://download.oracle.com/javase/6/docs/api/constant-values.html#java.lang.Float.MAX_EXPONENT case Types.FLOAT: return 24; // e.g. -(17#).e-### @@ -85,18 +134,25 @@ static int columnDisplaySize(int columnType) throws SQLException { case Types.DOUBLE: return 25; // e.g. -(17#).e-#### case Types.DECIMAL: + if (columnAttributes != null) { + return columnAttributes.precision + 2; // '-' sign and '.' + } return Integer.MAX_VALUE; default: throw new SQLException("Invalid column type: " + columnType); } } - static int columnPrecision(int columnType) throws SQLException { + static int columnPrecision(int columnType, ColumnAttributes columnAttributes) + throws SQLException { // according to hiveTypeToSqlType possible options are: switch(columnType) { case Types.BOOLEAN: return 1; case Types.VARCHAR: + if (columnAttributes != null) { + return columnAttributes.precision; + } return Integer.MAX_VALUE; // hive has no max limit for strings case Types.TINYINT: return 3; @@ -115,13 +171,17 @@ static int columnPrecision(int columnType) throws SQLException { case Types.TIMESTAMP: return 29; case Types.DECIMAL: + if (columnAttributes != null) { + return columnAttributes.precision; + } return Integer.MAX_VALUE; default: throw new SQLException("Invalid column type: " + columnType); } } - static int columnScale(int columnType) throws SQLException { + static int columnScale(int columnType, ColumnAttributes columnAttributes) + throws SQLException { // according to hiveTypeToSqlType possible options are: switch(columnType) { case Types.BOOLEAN: @@ -139,6 +199,9 @@ static int columnScale(int columnType) throws SQLException { case Types.TIMESTAMP: return 9; case Types.DECIMAL: + if (columnAttributes != null) { + return columnAttributes.scale; + } return Integer.MAX_VALUE; default: throw new SQLException("Invalid column type: " + columnType); @@ -146,13 +209,13 @@ static int columnScale(int columnType) throws SQLException { } public Integer getColumnSize() throws SQLException { - int precision = columnPrecision(Utils.hiveTypeToSqlType(type)); + int precision = columnPrecision(Utils.hiveTypeToSqlType(type), columnAttributes); return precision == 0 ? null : precision; } public Integer getDecimalDigits() throws SQLException { - return columnScale(Utils.hiveTypeToSqlType(type)); + return columnScale(Utils.hiveTypeToSqlType(type), columnAttributes); } public Integer getNumPrecRadix() { diff --git jdbc/src/java/org/apache/hadoop/hive/jdbc/Utils.java jdbc/src/java/org/apache/hadoop/hive/jdbc/Utils.java index bd98274..a1d9c74 100644 --- jdbc/src/java/org/apache/hadoop/hive/jdbc/Utils.java +++ jdbc/src/java/org/apache/hadoop/hive/jdbc/Utils.java @@ -52,6 +52,8 @@ public static int hiveTypeToSqlType(String type) throws SQLException { return Types.TIMESTAMP; } else if ("decimal".equalsIgnoreCase(type)) { return Types.DECIMAL; + } else if (type.startsWith("varchar")) { + return Types.VARCHAR; } else if (type.startsWith("map<")) { return Types.VARCHAR; } else if (type.startsWith("array<")) { diff --git jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java index f47df43..2daf46e 100644 --- jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java +++ jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java @@ -64,6 +64,7 @@ protected TRow row; protected List columnNames; protected List columnTypes; + protected List columnAttributes; private TableSchema schema; @@ -351,7 +352,7 @@ public long getLong(String columnName) throws SQLException { } public ResultSetMetaData getMetaData() throws SQLException { - return new HiveResultSetMetaData(columnNames, columnTypes); + return new HiveResultSetMetaData(columnNames, columnTypes, columnAttributes); } public Reader getNCharacterStream(int arg0) throws SQLException { @@ -509,6 +510,8 @@ private Object getColumnValue(int columnIndex) throws SQLException { return getDoubleValue(tColumnValue.getDoubleVal()); case STRING_TYPE: return getStringValue(tColumnValue.getStringVal()); + case VARCHAR_TYPE: + return getStringValue(tColumnValue.getStringVal()); case BINARY_TYPE: return getBinaryValue(tColumnValue.getStringVal()); case DATE_TYPE: diff --git jdbc/src/java/org/apache/hive/jdbc/HiveQueryResultSet.java jdbc/src/java/org/apache/hive/jdbc/HiveQueryResultSet.java index 61985d1..609636d 100644 --- jdbc/src/java/org/apache/hive/jdbc/HiveQueryResultSet.java +++ jdbc/src/java/org/apache/hive/jdbc/HiveQueryResultSet.java @@ -29,7 +29,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hive.service.cli.TableSchema; +import org.apache.hive.service.cli.TypeQualifiers; import org.apache.hive.service.cli.thrift.TCLIService; +import org.apache.hive.service.cli.thrift.TCLIServiceConstants; import org.apache.hive.service.cli.thrift.TColumnDesc; import org.apache.hive.service.cli.thrift.TFetchOrientation; import org.apache.hive.service.cli.thrift.TFetchResultsReq; @@ -37,9 +39,12 @@ import org.apache.hive.service.cli.thrift.TGetResultSetMetadataReq; import org.apache.hive.service.cli.thrift.TGetResultSetMetadataResp; import org.apache.hive.service.cli.thrift.TOperationHandle; +import org.apache.hive.service.cli.thrift.TPrimitiveTypeEntry; import org.apache.hive.service.cli.thrift.TRow; import org.apache.hive.service.cli.thrift.TSessionHandle; import org.apache.hive.service.cli.thrift.TTableSchema; +import org.apache.hive.service.cli.thrift.TTypeQualifierValue; +import org.apache.hive.service.cli.thrift.TTypeQualifiers; /** * HiveQueryResultSet. @@ -76,6 +81,7 @@ private boolean retrieveSchema = true; private List colNames; private List colTypes; + private List colAttributes; private int fetchSize = 50; private boolean emptyResultSet = false; @@ -100,10 +106,23 @@ public Builder setMaxRows(int maxRows) { } public Builder setSchema(List colNames, List colTypes) { + // no column attributes provided - create list of null attributes. + List colAttributes = + new ArrayList(); + for (int idx = 0; idx < colTypes.size(); ++idx) { + colAttributes.add(null); + } + return setSchema(colNames, colTypes, colAttributes); + } + + public Builder setSchema(List colNames, List colTypes, + List colAttributes) { this.colNames = new ArrayList(); this.colNames.addAll(colNames); this.colTypes = new ArrayList(); this.colTypes.addAll(colTypes); + this.colAttributes = new ArrayList(); + this.colAttributes.addAll(colAttributes); this.retrieveSchema = false; return this; } @@ -130,6 +149,7 @@ protected HiveQueryResultSet(Builder builder) throws SQLException { this.fetchSize = builder.fetchSize; columnNames = new ArrayList(); columnTypes = new ArrayList(); + columnAttributes = new ArrayList(); if (builder.retrieveSchema) { retrieveSchema(); } else { @@ -145,6 +165,32 @@ protected HiveQueryResultSet(Builder builder) throws SQLException { } /** + * Generate ColumnAttributes object from a TTypeQualifiers + * @param primitiveTypeEntry primitive type + * @return generated ColumnAttributes, or null + */ + private static JdbcColumn.ColumnAttributes getColumnAttributes( + TPrimitiveTypeEntry primitiveTypeEntry) { + JdbcColumn.ColumnAttributes ret = null; + if (primitiveTypeEntry.isSetTypeQualifiers()) { + TTypeQualifiers tq = primitiveTypeEntry.getTypeQualifiers(); + switch (primitiveTypeEntry.getType()) { + case VARCHAR_TYPE: + TTypeQualifierValue val = + tq.getQualifiers().get(TCLIServiceConstants.CHARACTER_MAXIMUM_LENGTH); + if (val != null) { + // precision is char length + ret = new JdbcColumn.ColumnAttributes(val.getI32Value(), 0); + } + break; + default: + break; + } + } + return ret; + } + + /** * Retrieve schema from the server */ private void retrieveSchema() throws SQLException { @@ -172,9 +218,11 @@ private void retrieveSchema() throws SQLException { } String columnName = columns.get(pos).getColumnName(); columnNames.add(columnName); - String columnTypeName = TYPE_NAMES.get( - columns.get(pos).getTypeDesc().getTypes().get(0).getPrimitiveEntry().getType()); + TPrimitiveTypeEntry primitiveTypeEntry = + columns.get(pos).getTypeDesc().getTypes().get(0).getPrimitiveEntry(); + String columnTypeName = TYPE_NAMES.get(primitiveTypeEntry.getType()); columnTypes.add(columnTypeName); + columnAttributes.add(getColumnAttributes(primitiveTypeEntry)); } } catch (SQLException eS) { throw eS; // rethrow the SQLException as is @@ -189,9 +237,11 @@ private void retrieveSchema() throws SQLException { * @param colNames * @param colTypes */ - public void setSchema(List colNames, List colTypes) { + public void setSchema(List colNames, List colTypes, + List colAttributes) { columnNames.addAll(colNames); columnTypes.addAll(colTypes); + columnAttributes.addAll(colAttributes); } @Override diff --git jdbc/src/java/org/apache/hive/jdbc/HiveResultSetMetaData.java jdbc/src/java/org/apache/hive/jdbc/HiveResultSetMetaData.java index ec6fee8..ceed72a 100644 --- jdbc/src/java/org/apache/hive/jdbc/HiveResultSetMetaData.java +++ jdbc/src/java/org/apache/hive/jdbc/HiveResultSetMetaData.java @@ -20,6 +20,7 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.util.Arrays; import java.util.List; import org.apache.hadoop.hive.serde.serdeConstants; @@ -31,11 +32,14 @@ public class HiveResultSetMetaData implements java.sql.ResultSetMetaData { private final List columnNames; private final List columnTypes; + private final List columnAttributes; public HiveResultSetMetaData(List columnNames, - List columnTypes) { + List columnTypes, + List columnAttributes) { this.columnNames = columnNames; this.columnTypes = columnTypes; + this.columnAttributes = columnAttributes; } public String getCatalogName(int column) throws SQLException { @@ -53,7 +57,7 @@ public int getColumnCount() throws SQLException { public int getColumnDisplaySize(int column) throws SQLException { int columnType = getColumnType(column); - return JdbcColumn.columnDisplaySize(columnType); + return JdbcColumn.columnDisplaySize(columnType, columnAttributes.get(column - 1)); } public String getColumnLabel(int column) throws SQLException { @@ -89,6 +93,8 @@ public String getColumnTypeName(int column) throws SQLException { String type = columnTypes.get(column - 1); if ("string".equalsIgnoreCase(type)) { return serdeConstants.STRING_TYPE_NAME; + } else if ("varchar".equalsIgnoreCase(type)) { + return serdeConstants.VARCHAR_TYPE_NAME; } else if ("float".equalsIgnoreCase(type)) { return serdeConstants.FLOAT_TYPE_NAME; } else if ("double".equalsIgnoreCase(type)) { @@ -127,13 +133,13 @@ public String getColumnTypeName(int column) throws SQLException { public int getPrecision(int column) throws SQLException { int columnType = getColumnType(column); - return JdbcColumn.columnPrecision(columnType); + return JdbcColumn.columnPrecision(columnType, columnAttributes.get(column - 1)); } public int getScale(int column) throws SQLException { int columnType = getColumnType(column); - return JdbcColumn.columnScale(columnType); + return JdbcColumn.columnScale(columnType, columnAttributes.get(column - 1)); } public String getSchemaName(int column) throws SQLException { diff --git jdbc/src/java/org/apache/hive/jdbc/JdbcColumn.java jdbc/src/java/org/apache/hive/jdbc/JdbcColumn.java index 5553d7d..7760713 100644 --- jdbc/src/java/org/apache/hive/jdbc/JdbcColumn.java +++ jdbc/src/java/org/apache/hive/jdbc/JdbcColumn.java @@ -33,6 +33,19 @@ private final String comment; private final int ordinalPos; + static class ColumnAttributes { + public int precision = 0; + public int scale = 0; + + public ColumnAttributes() { + } + + public ColumnAttributes(int precision, int scale) { + this.precision = precision; + this.scale = scale; + } + } + JdbcColumn(String columnName, String tableName, String tableCatalog , String type, String comment, int ordinalPos) { this.columnName = columnName; @@ -63,22 +76,26 @@ public Integer getSqlType() throws SQLException { return Utils.hiveTypeToSqlType(type); } - static int columnDisplaySize(int columnType) throws SQLException { + static int columnDisplaySize(int columnType, ColumnAttributes columnAttributes) + throws SQLException { // according to hiveTypeToSqlType possible options are: switch(columnType) { case Types.BOOLEAN: - return columnPrecision(columnType); + return columnPrecision(columnType, columnAttributes); case Types.VARCHAR: + if (columnAttributes != null) { + return columnAttributes.precision; + } return Integer.MAX_VALUE; // hive has no max limit for strings case Types.TINYINT: case Types.SMALLINT: case Types.INTEGER: case Types.BIGINT: - return columnPrecision(columnType) + 1; // allow +/- + return columnPrecision(columnType, columnAttributes) + 1; // allow +/- case Types.DATE: return 10; case Types.TIMESTAMP: - return columnPrecision(columnType); + return columnPrecision(columnType, columnAttributes); // see http://download.oracle.com/javase/6/docs/api/constant-values.html#java.lang.Float.MAX_EXPONENT case Types.FLOAT: @@ -87,18 +104,25 @@ static int columnDisplaySize(int columnType) throws SQLException { case Types.DOUBLE: return 25; // e.g. -(17#).e-#### case Types.DECIMAL: + if (columnAttributes != null) { + return columnAttributes.precision + 2; // '-' sign and '.' + } return Integer.MAX_VALUE; default: throw new SQLException("Invalid column type: " + columnType); } } - static int columnPrecision(int columnType) throws SQLException { + static int columnPrecision(int columnType, ColumnAttributes columnAttributes) + throws SQLException { // according to hiveTypeToSqlType possible options are: switch(columnType) { case Types.BOOLEAN: return 1; case Types.VARCHAR: + if (columnAttributes != null) { + return columnAttributes.precision; + } return Integer.MAX_VALUE; // hive has no max limit for strings case Types.TINYINT: return 3; @@ -117,13 +141,17 @@ static int columnPrecision(int columnType) throws SQLException { case Types.TIMESTAMP: return 29; case Types.DECIMAL: + if (columnAttributes != null) { + return columnAttributes.precision; + } return Integer.MAX_VALUE; default: throw new SQLException("Invalid column type: " + columnType); } } - static int columnScale(int columnType) throws SQLException { + static int columnScale(int columnType, ColumnAttributes columnAttributes) + throws SQLException { // according to hiveTypeToSqlType possible options are: switch(columnType) { case Types.BOOLEAN: @@ -141,6 +169,9 @@ static int columnScale(int columnType) throws SQLException { case Types.TIMESTAMP: return 9; case Types.DECIMAL: + if (columnAttributes != null) { + return columnAttributes.scale; + } return Integer.MAX_VALUE; default: throw new SQLException("Invalid column type: " + columnType); @@ -148,13 +179,13 @@ static int columnScale(int columnType) throws SQLException { } public Integer getColumnSize() throws SQLException { - int precision = columnPrecision(Utils.hiveTypeToSqlType(type)); + int precision = columnPrecision(Utils.hiveTypeToSqlType(type), null); return precision == 0 ? null : precision; } public Integer getDecimalDigits() throws SQLException { - return columnScale(Utils.hiveTypeToSqlType(type)); + return columnScale(Utils.hiveTypeToSqlType(type), null); } public Integer getNumPrecRadix() { diff --git jdbc/src/java/org/apache/hive/jdbc/Utils.java jdbc/src/java/org/apache/hive/jdbc/Utils.java index 3df3bd7..754e107 100644 --- jdbc/src/java/org/apache/hive/jdbc/Utils.java +++ jdbc/src/java/org/apache/hive/jdbc/Utils.java @@ -114,6 +114,8 @@ public void setSessionVars(Map sessionVars) { public static int hiveTypeToSqlType(String type) throws SQLException { if ("string".equalsIgnoreCase(type)) { return Types.VARCHAR; + } else if ("varchar".equalsIgnoreCase(type)) { + return Types.VARCHAR; } else if ("float".equalsIgnoreCase(type)) { return Types.FLOAT; } else if ("double".equalsIgnoreCase(type)) { diff --git jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java index e1107dd..4375cbf 100644 --- jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java +++ jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java @@ -152,7 +152,9 @@ protected void setUp() throws Exception { + " c17 timestamp, " + " c18 decimal," + " c19 binary," - + " c20 date) comment'" + dataTypeTableComment + + " c20 date," + + " c21 varchar(20)" + + ") comment'" + dataTypeTableComment +"' partitioned by (dt STRING)"); assertFalse(res.next()); @@ -445,6 +447,7 @@ public void testDataTypes() throws Exception { assertEquals(null, res.getBigDecimal(18)); assertEquals(null, res.getString(20)); assertEquals(null, res.getDate(20)); + assertEquals(null, res.getString(21)); // row 3 assertTrue(res.next()); @@ -469,6 +472,7 @@ public void testDataTypes() throws Exception { assertEquals("123456789.0123456", res.getBigDecimal(18).toString()); assertEquals("2013-01-01", res.getString(20)); assertEquals("2013-01-01", res.getDate(20).toString()); + assertEquals("abc123", res.getString(21)); // test getBoolean rules on non-boolean columns assertEquals(true, res.getBoolean(1)); @@ -859,14 +863,14 @@ public void testResultSetMetaData() throws SQLException { ResultSet res = stmt.executeQuery( "select c1, c2, c3, c4, c5 as a, c6, c7, c8, c9, c10, c11, c12, " + - "c1*2, sentences(null, null, null) as b, c17, c18, c20 from " + dataTypeTableName + + "c1*2, sentences(null, null, null) as b, c17, c18, c20, c21 from " + dataTypeTableName + " limit 1"); ResultSetMetaData meta = res.getMetaData(); ResultSet colRS = con.getMetaData().getColumns(null, null, dataTypeTableName.toLowerCase(), null); - assertEquals(17, meta.getColumnCount()); + assertEquals(18, meta.getColumnCount()); assertTrue(colRS.next()); @@ -1083,6 +1087,14 @@ public void testResultSetMetaData() throws SQLException { assertEquals(10, meta.getPrecision(17)); assertEquals(0, meta.getScale(17)); + assertEquals("c21", meta.getColumnName(18)); + assertEquals(Types.VARCHAR, meta.getColumnType(18)); + assertEquals("varchar", meta.getColumnTypeName(18)); + // varchar columns should have correct display size/precision + assertEquals(20, meta.getColumnDisplaySize(18)); + assertEquals(20, meta.getPrecision(18)); + assertEquals(0, meta.getScale(18)); + for (int i = 1; i <= meta.getColumnCount(); i++) { assertFalse(meta.isAutoIncrement(i)); assertFalse(meta.isCurrency(i)); diff --git jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java index 601bcb3..41ebc76 100644 --- jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java +++ jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java @@ -156,7 +156,9 @@ protected void setUp() throws Exception { + " c17 timestamp, " + " c18 decimal, " + " c19 binary, " - + " c20 date) comment'" + dataTypeTableComment + + " c20 date," + + " c21 varchar(20)" + + ") comment'" + dataTypeTableComment +"' partitioned by (dt STRING)"); stmt.execute("load data local inpath '" @@ -495,6 +497,7 @@ public void testDataTypes() throws Exception { assertEquals(null, res.getString(19)); assertEquals(null, res.getString(20)); assertEquals(null, res.getDate(20)); + assertEquals(null, res.getString(21)); // row 2 assertTrue(res.next()); @@ -520,6 +523,7 @@ public void testDataTypes() throws Exception { assertEquals(null, res.getString(19)); assertEquals(null, res.getString(20)); assertEquals(null, res.getDate(20)); + assertEquals(null, res.getString(21)); // row 3 assertTrue(res.next()); @@ -545,6 +549,7 @@ public void testDataTypes() throws Exception { assertEquals("abcd", res.getString(19)); assertEquals("2013-01-01", res.getString(20)); assertEquals("2013-01-01", res.getDate(20).toString()); + assertEquals("abc123", res.getString(21)); // test getBoolean rules on non-boolean columns assertEquals(true, res.getBoolean(1)); @@ -989,14 +994,14 @@ public void testResultSetMetaData() throws SQLException { ResultSet res = stmt.executeQuery( "select c1, c2, c3, c4, c5 as a, c6, c7, c8, c9, c10, c11, c12, " + - "c1*2, sentences(null, null, null) as b, c17, c18, c20 from " + dataTypeTableName + + "c1*2, sentences(null, null, null) as b, c17, c18, c20, c21 from " + dataTypeTableName + " limit 1"); ResultSetMetaData meta = res.getMetaData(); ResultSet colRS = con.getMetaData().getColumns(null, null, dataTypeTableName.toLowerCase(), null); - assertEquals(17, meta.getColumnCount()); + assertEquals(18, meta.getColumnCount()); assertTrue(colRS.next()); @@ -1204,6 +1209,14 @@ public void testResultSetMetaData() throws SQLException { assertEquals(10, meta.getPrecision(17)); assertEquals(0, meta.getScale(17)); + assertEquals("c21", meta.getColumnName(18)); + assertEquals(Types.VARCHAR, meta.getColumnType(18)); + assertEquals("varchar", meta.getColumnTypeName(18)); + // varchar columns should have correct display size/precision + assertEquals(20, meta.getColumnDisplaySize(18)); + assertEquals(20, meta.getPrecision(18)); + assertEquals(0, meta.getScale(18)); + for (int i = 1; i <= meta.getColumnCount(); i++) { assertFalse(meta.isAutoIncrement(i)); assertFalse(meta.isCurrency(i)); diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index 15a2a81..c9c09a2 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -65,6 +65,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.shims.ShimLoader; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge; public class MetaStoreUtils { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java index b28d16e..7c1c344 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java @@ -56,6 +56,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.ParameterizedPrimitiveTypeUtils; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapred.InputFormat; @@ -130,7 +131,9 @@ private void initialize() { List inspectors = new ArrayList(vcCols.size()); for (VirtualColumn vc : vcCols) { inspectors.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( - vc.getTypeInfo().getPrimitiveCategory())); + vc.getTypeInfo().getPrimitiveCategory(), + ParameterizedPrimitiveTypeUtils.getTypeParamsFromTypeInfo( + vc.getTypeInfo()))); names.add(vc.getName()); } vcsOI = ObjectInspectorFactory.getStandardStructObjectInspector(names, inspectors); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index 03cf85d..9c99ec3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -23,6 +23,7 @@ import java.net.URL; import java.util.ArrayList; import java.util.Collections; +import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -39,6 +40,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -54,7 +56,6 @@ import org.apache.hadoop.hive.ql.udf.UDFBase64; import org.apache.hadoop.hive.ql.udf.UDFBin; import org.apache.hadoop.hive.ql.udf.UDFCeil; -import org.apache.hadoop.hive.ql.udf.UDFConcat; import org.apache.hadoop.hive.ql.udf.UDFConv; import org.apache.hadoop.hive.ql.udf.UDFCos; import org.apache.hadoop.hive.ql.udf.UDFDate; @@ -78,7 +79,6 @@ import org.apache.hadoop.hive.ql.udf.UDFLog; import org.apache.hadoop.hive.ql.udf.UDFLog10; import org.apache.hadoop.hive.ql.udf.UDFLog2; -import org.apache.hadoop.hive.ql.udf.UDFLower; import org.apache.hadoop.hive.ql.udf.UDFLpad; import org.apache.hadoop.hive.ql.udf.UDFMinute; import org.apache.hadoop.hive.ql.udf.UDFMonth; @@ -127,7 +127,6 @@ import org.apache.hadoop.hive.ql.udf.UDFType; import org.apache.hadoop.hive.ql.udf.UDFUnbase64; import org.apache.hadoop.hive.ql.udf.UDFUnhex; -import org.apache.hadoop.hive.ql.udf.UDFUpper; import org.apache.hadoop.hive.ql.udf.UDFWeekOfYear; import org.apache.hadoop.hive.ql.udf.UDFYear; import org.apache.hadoop.hive.ql.udf.generic.*; @@ -149,11 +148,16 @@ import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeParams; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.ReflectionUtils; import org.w3c.dom.Document; @@ -190,7 +194,7 @@ static { - registerUDF("concat", UDFConcat.class, false); + registerGenericUDF("concat", GenericUDFConcat.class); registerUDF("substr", UDFSubstr.class, false); registerUDF("substring", UDFSubstr.class, false); registerUDF("space", UDFSpace.class, false); @@ -239,10 +243,10 @@ registerGenericUDF("encode", GenericUDFEncode.class); registerGenericUDF("decode", GenericUDFDecode.class); - registerUDF("upper", UDFUpper.class, false); - registerUDF("lower", UDFLower.class, false); - registerUDF("ucase", UDFUpper.class, false); - registerUDF("lcase", UDFLower.class, false); + registerGenericUDF("upper", GenericUDFUpper.class); + registerGenericUDF("lower", GenericUDFLower.class); + registerGenericUDF("ucase", GenericUDFUpper.class); + registerGenericUDF("lcase", GenericUDFLower.class); registerUDF("trim", UDFTrim.class, false); registerUDF("ltrim", UDFLTrim.class, false); registerUDF("rtrim", UDFRTrim.class, false); @@ -356,6 +360,8 @@ GenericUDFToBinary.class); registerGenericUDF(serdeConstants.DECIMAL_TYPE_NAME, GenericUDFToDecimal.class); + registerGenericUDF(serdeConstants.VARCHAR_TYPE_NAME, + GenericUDFToVarchar.class); // Aggregate functions registerGenericUDAF("max", new GenericUDAFMax()); @@ -597,24 +603,89 @@ public static FunctionInfo getFunctionInfo(String functionName) { return synonyms; } - static Map numericTypes = new HashMap(); - static List numericTypeList = new ArrayList(); + // The ordering of types here is used to determine which numeric types + // are common/convertible to one another. Probably better to rely on the + // ordering explicitly defined here than to assume that the enum values + // that were arbitrarily assigned in PrimitiveCategory work for our purposes. + static EnumMap numericTypes = + new EnumMap(PrimitiveCategory.class); + static List numericTypeList = new ArrayList(); - static void registerNumericType(String typeName, int level) { - TypeInfo t = TypeInfoFactory.getPrimitiveTypeInfo(typeName); - numericTypeList.add(t); - numericTypes.put(t, level); + static void registerNumericType(PrimitiveCategory primitiveCategory, int level) { + numericTypeList.add(primitiveCategory); + numericTypes.put(primitiveCategory, level); } static { - registerNumericType(serdeConstants.TINYINT_TYPE_NAME, 1); - registerNumericType(serdeConstants.SMALLINT_TYPE_NAME, 2); - registerNumericType(serdeConstants.INT_TYPE_NAME, 3); - registerNumericType(serdeConstants.BIGINT_TYPE_NAME, 4); - registerNumericType(serdeConstants.FLOAT_TYPE_NAME, 5); - registerNumericType(serdeConstants.DOUBLE_TYPE_NAME, 6); - registerNumericType(serdeConstants.DECIMAL_TYPE_NAME, 7); - registerNumericType(serdeConstants.STRING_TYPE_NAME, 8); + registerNumericType(PrimitiveCategory.BYTE, 1); + registerNumericType(PrimitiveCategory.SHORT, 2); + registerNumericType(PrimitiveCategory.INT, 3); + registerNumericType(PrimitiveCategory.LONG, 4); + registerNumericType(PrimitiveCategory.FLOAT, 5); + registerNumericType(PrimitiveCategory.DOUBLE, 6); + registerNumericType(PrimitiveCategory.DECIMAL, 7); + registerNumericType(PrimitiveCategory.STRING, 8); + } + + static int getCharacterLengthForType(PrimitiveTypeInfo typeInfo) { + switch (typeInfo.getPrimitiveCategory()) { + case STRING: + return HiveVarchar.MAX_VARCHAR_LENGTH; + case VARCHAR: + VarcharTypeParams varcharParams = (VarcharTypeParams) typeInfo.getParameters(); + if (varcharParams == null) { + throw new RuntimeException("varchar type used without type params"); + } + return varcharParams.getLength(); + default: + return 0; + } + } + + static int getCommonLength(int aLen, int bLen) { + int maxLength; + if (aLen < 0 || bLen < 0) { + // negative length should take precedence over positive value? + maxLength = -1; + } else { + maxLength = Math.max(aLen, bLen); + } + return maxLength; + } + + /** + * Given 2 TypeInfo types and the PrimitiveCategory selected as the common class between the two, + * return a TypeInfo corresponding to the common PrimitiveCategory, and with type qualifiers + * (if applicable) that match the 2 TypeInfo types. + * Examples: + * varchar(10), varchar(20), primitive category varchar => varchar(20) + * date, string, primitive category string => string + * @param a TypeInfo of the first type + * @param b TypeInfo of the second type + * @param typeCategory PrimitiveCategory of the designated common type between a and b + * @return TypeInfo represented by the primitive category, with any applicable type qualifiers. + */ + public static TypeInfo getTypeInfoForPrimitiveCategory( + PrimitiveTypeInfo a, PrimitiveTypeInfo b, PrimitiveCategory typeCategory) { + // For types with parameters (like varchar), we need to determine the type parameters + // that should be added to this type, based on the original 2 TypeInfos. + switch (typeCategory) { + case VARCHAR: + int maxLength = getCommonLength( + getCharacterLengthForType(a), + getCharacterLengthForType(b)); + VarcharTypeParams varcharParams = new VarcharTypeParams(); + varcharParams.setLength(maxLength); + String typeName = + PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveCategory(typeCategory).typeName + + varcharParams.toString(); + return TypeInfoFactory.getPrimitiveTypeInfo(typeName); + + default: + // Type doesn't require any qualifiers. + return TypeInfoFactory.getPrimitiveTypeInfo( + PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveCategory(typeCategory).typeName); + } } /** @@ -624,18 +695,38 @@ public static TypeInfo getCommonClassForUnionAll(TypeInfo a, TypeInfo b) { if (a.equals(b)) { return a; } + if (a.getCategory() != Category.PRIMITIVE || b.getCategory() != Category.PRIMITIVE) { + return null; + } + PrimitiveCategory pcA = ((PrimitiveTypeInfo)a).getPrimitiveCategory(); + PrimitiveCategory pcB = ((PrimitiveTypeInfo)b).getPrimitiveCategory(); + + if (pcA == pcB) { + // Same primitive category but different qualifiers. + return getTypeInfoForPrimitiveCategory((PrimitiveTypeInfo)a, (PrimitiveTypeInfo)b, pcA); + } + + PrimitiveGrouping pgA = PrimitiveObjectInspectorUtils.getPrimitiveGrouping(pcA); + PrimitiveGrouping pgB = PrimitiveObjectInspectorUtils.getPrimitiveGrouping(pcB); + // handle string types properly + if (pgA == PrimitiveGrouping.STRING_GROUP && pgB == PrimitiveGrouping.STRING_GROUP) { + return getTypeInfoForPrimitiveCategory( + (PrimitiveTypeInfo)a, (PrimitiveTypeInfo)b,PrimitiveCategory.STRING); + } + if (FunctionRegistry.implicitConvertable(a, b)) { - return b; + return getTypeInfoForPrimitiveCategory((PrimitiveTypeInfo)a, (PrimitiveTypeInfo)b, pcB); } if (FunctionRegistry.implicitConvertable(b, a)) { - return a; + return getTypeInfoForPrimitiveCategory((PrimitiveTypeInfo)a, (PrimitiveTypeInfo)b, pcA); } - for (TypeInfo t : numericTypeList) { - if (FunctionRegistry.implicitConvertable(a, t) - && FunctionRegistry.implicitConvertable(b, t)) { - return t; + for (PrimitiveCategory t : numericTypeList) { + if (FunctionRegistry.implicitConvertable(pcA, t) + && FunctionRegistry.implicitConvertable(pcB, t)) { + return getTypeInfoForPrimitiveCategory((PrimitiveTypeInfo)a, (PrimitiveTypeInfo)b, t); } } + return null; } @@ -653,12 +744,34 @@ public static TypeInfo getCommonClassForComparison(TypeInfo a, TypeInfo b) { if (a.equals(b)) { return a; } - for (TypeInfo t : numericTypeList) { - if (FunctionRegistry.implicitConvertable(a, t) - && FunctionRegistry.implicitConvertable(b, t)) { - return t; + if (a.getCategory() != Category.PRIMITIVE || b.getCategory() != Category.PRIMITIVE) { + return null; + } + PrimitiveCategory pcA = ((PrimitiveTypeInfo)a).getPrimitiveCategory(); + PrimitiveCategory pcB = ((PrimitiveTypeInfo)b).getPrimitiveCategory(); + + if (pcA == pcB) { + // Same primitive category but different qualifiers. + // Rely on getTypeInfoForPrimitiveCategory() to sort out the type params. + return getTypeInfoForPrimitiveCategory((PrimitiveTypeInfo)a, (PrimitiveTypeInfo)b, pcA); + } + + PrimitiveGrouping pgA = PrimitiveObjectInspectorUtils.getPrimitiveGrouping(pcA); + PrimitiveGrouping pgB = PrimitiveObjectInspectorUtils.getPrimitiveGrouping(pcB); + // handle string types properly + if (pgA == PrimitiveGrouping.STRING_GROUP && pgB == PrimitiveGrouping.STRING_GROUP) { + // Compare as strings. Char comparison semantics may be different if/when implemented. + return getTypeInfoForPrimitiveCategory( + (PrimitiveTypeInfo)a, (PrimitiveTypeInfo)b,PrimitiveCategory.STRING); + } + + for (PrimitiveCategory t : numericTypeList) { + if (FunctionRegistry.implicitConvertable(pcA, t) + && FunctionRegistry.implicitConvertable(pcB, t)) { + return getTypeInfoForPrimitiveCategory((PrimitiveTypeInfo)a, (PrimitiveTypeInfo)b, t); } } + return null; } @@ -674,45 +787,70 @@ public static TypeInfo getCommonClass(TypeInfo a, TypeInfo b) { if (a.equals(b)) { return a; } - Integer ai = numericTypes.get(a); - Integer bi = numericTypes.get(b); + if (a.getCategory() != Category.PRIMITIVE || b.getCategory() != Category.PRIMITIVE) { + return null; + } + PrimitiveCategory pcA = ((PrimitiveTypeInfo)a).getPrimitiveCategory(); + PrimitiveCategory pcB = ((PrimitiveTypeInfo)b).getPrimitiveCategory(); + + PrimitiveGrouping pgA = PrimitiveObjectInspectorUtils.getPrimitiveGrouping(pcA); + PrimitiveGrouping pgB = PrimitiveObjectInspectorUtils.getPrimitiveGrouping(pcB); + // handle string types properly + if (pgA == PrimitiveGrouping.STRING_GROUP && pgB == PrimitiveGrouping.STRING_GROUP) { + return getTypeInfoForPrimitiveCategory( + (PrimitiveTypeInfo)a, (PrimitiveTypeInfo)b,PrimitiveCategory.STRING); + } + + Integer ai = numericTypes.get(pcA); + Integer bi = numericTypes.get(pcB); if (ai == null || bi == null) { // If either is not a numeric type, return null. return null; } - return (ai > bi) ? a : b; + PrimitiveCategory pcCommon = (ai > bi) ? pcA : pcB; + return getTypeInfoForPrimitiveCategory((PrimitiveTypeInfo)a, (PrimitiveTypeInfo)b, pcCommon); } - /** - * Returns whether it is possible to implicitly convert an object of Class - * from to Class to. - */ - public static boolean implicitConvertable(TypeInfo from, TypeInfo to) { - if (from.equals(to)) { + public static boolean isConversionRequiredForComparison(TypeInfo typeA, TypeInfo typeB) { + if (typeA == typeB) { + return false; + } + if (TypeInfoUtils.doPrimitiveCategoriesMatch(typeA, typeB)) { + return false; + } + return true; + } + + public static boolean implicitConvertable(PrimitiveCategory from, PrimitiveCategory to) { + if (from == to) { return true; } + + PrimitiveGrouping fromPg = PrimitiveObjectInspectorUtils.getPrimitiveGrouping(from); + PrimitiveGrouping toPg = PrimitiveObjectInspectorUtils.getPrimitiveGrouping(to); + // Allow implicit String to Double conversion - if (from.equals(TypeInfoFactory.stringTypeInfo) - && to.equals(TypeInfoFactory.doubleTypeInfo)) { + if (fromPg == PrimitiveGrouping.STRING_GROUP && to == PrimitiveCategory.DOUBLE) { return true; } // Allow implicit String to Decimal conversion - if (from.equals(TypeInfoFactory.stringTypeInfo) - && to.equals(TypeInfoFactory.decimalTypeInfo)) { + if (fromPg == PrimitiveGrouping.STRING_GROUP && to == PrimitiveCategory.DECIMAL) { return true; } // Void can be converted to any type - if (from.equals(TypeInfoFactory.voidTypeInfo)) { + if (from == PrimitiveCategory.VOID) { return true; } // Allow implicit String to Date conversion - if (from.equals(TypeInfoFactory.dateTypeInfo) - && to.equals(TypeInfoFactory.stringTypeInfo)) { + if (fromPg == PrimitiveGrouping.DATE_GROUP && toPg == PrimitiveGrouping.STRING_GROUP) { return true; } - - if (from.equals(TypeInfoFactory.timestampTypeInfo) - && to.equals(TypeInfoFactory.stringTypeInfo)) { + // Allow implicit Numeric to String conversion + if (fromPg == PrimitiveGrouping.NUMERIC_GROUP && toPg == PrimitiveGrouping.STRING_GROUP) { + return true; + } + // Allow implicit String to varchar conversion, and vice versa + if (fromPg == PrimitiveGrouping.STRING_GROUP && toPg == PrimitiveGrouping.STRING_GROUP) { return true; } @@ -730,6 +868,26 @@ public static boolean implicitConvertable(TypeInfo from, TypeInfo to) { } /** + * Returns whether it is possible to implicitly convert an object of Class + * from to Class to. + */ + public static boolean implicitConvertable(TypeInfo from, TypeInfo to) { + if (from.equals(to)) { + return true; + } + + // Reimplemented to use PrimitiveCategory rather than TypeInfo, because + // 2 TypeInfos from the same qualified type (varchar, decimal) should still be + // seen as equivalent. + if (from.getCategory() == Category.PRIMITIVE && to.getCategory() == Category.PRIMITIVE) { + return implicitConvertable( + ((PrimitiveTypeInfo)from).getPrimitiveCategory(), + ((PrimitiveTypeInfo)to).getPrimitiveCategory()); + } + return false; + } + + /** * Get the GenericUDAF evaluator for the name and argumentClasses. * * @param name @@ -907,7 +1065,8 @@ public static Object invoke(Method m, Object thisObject, Object... arguments) */ public static int matchCost(TypeInfo argumentPassed, TypeInfo argumentAccepted, boolean exact) { - if (argumentAccepted.equals(argumentPassed)) { + if (argumentAccepted.equals(argumentPassed) + || TypeInfoUtils.doPrimitiveCategoriesMatch(argumentPassed, argumentAccepted)) { // matches return 0; } @@ -1025,6 +1184,50 @@ public static Method getMethodInternal(Class udfClass, List mlist, bo // No matching methods found throw new NoMatchingMethodException(udfClass, argumentsPassed, mlist); } + + if (udfMethods.size() > 1) { + // Prefer methods with a closer signature based on the primitive grouping of each argument. + // For a varchar argument, we would prefer evaluate(string) over evaluate(double). + int currentScore = 0; + int bestMatchScore = 0; + Method bestMatch = null; + for (Method m: udfMethods) { + currentScore = 0; + List argumentsAccepted = + TypeInfoUtils.getParameterTypeInfos(m, argumentsPassed.size()); + Iterator argsPassedIter = argumentsPassed.iterator(); + for (TypeInfo acceptedType : argumentsAccepted) { + // Check the affinity of the argument passed in with the accepted argument, + // based on the PrimitiveGrouping + TypeInfo passedType = argsPassedIter.next(); + if (acceptedType.getCategory() == Category.PRIMITIVE + && passedType.getCategory() == Category.PRIMITIVE) { + PrimitiveGrouping acceptedPg = PrimitiveObjectInspectorUtils.getPrimitiveGrouping( + ((PrimitiveTypeInfo) acceptedType).getPrimitiveCategory()); + PrimitiveGrouping passedPg = PrimitiveObjectInspectorUtils.getPrimitiveGrouping( + ((PrimitiveTypeInfo) passedType).getPrimitiveCategory()); + if (acceptedPg == passedPg) { + // The passed argument matches somewhat closely with an accepted argument + ++currentScore; + } + } + } + // Check if the score for this method is any better relative to others + if (currentScore > bestMatchScore) { + bestMatchScore = currentScore; + bestMatch = m; + } else if (currentScore == bestMatchScore) { + bestMatch = null; // no longer a best match if more than one. + } + } + + if (bestMatch != null) { + // Found a best match during this processing, use it. + udfMethods.clear(); + udfMethods.add(bestMatch); + } + } + if (udfMethods.size() > 1) { // if the only difference is numeric types, pick the method @@ -1050,9 +1253,15 @@ public static Method getMethodInternal(Class udfClass, List mlist, bo for (TypeInfo accepted: argumentsAccepted) { TypeInfo reference = referenceIterator.next(); - if (numericTypes.containsKey(accepted)) { + boolean acceptedIsPrimitive = false; + PrimitiveCategory acceptedPrimCat = PrimitiveCategory.UNKNOWN; + if (accepted.getCategory() == Category.PRIMITIVE) { + acceptedIsPrimitive = true; + acceptedPrimCat = ((PrimitiveTypeInfo) accepted).getPrimitiveCategory(); + } + if (acceptedIsPrimitive && numericTypes.containsKey(acceptedPrimCat)) { // We're looking for the udf with the smallest maximum numeric type. - int typeValue = numericTypes.get(accepted); + int typeValue = numericTypes.get(acceptedPrimCat); maxNumericType = typeValue > maxNumericType ? typeValue : maxNumericType; } else if (!accepted.equals(reference)) { // There are non-numeric arguments that don't match from one UDF to @@ -1114,8 +1323,20 @@ public static GenericUDF cloneGenericUDF(GenericUDF genericUDF) { bridge.getColNames(), bridge.getColTypes()); } - return (GenericUDF) ReflectionUtils + GenericUDF newInstance = (GenericUDF) ReflectionUtils .newInstance(genericUDF.getClass(), null); + // The original may have settable info that needs to be added to the new copy. + if (genericUDF instanceof SettableUDF) { + try { + Object settableData = ((SettableUDF)genericUDF).getParams(); + if (settableData != null) { + ((SettableUDF)newInstance).setParams(settableData); + } + } catch (UDFArgumentException err) { + LOG.error("Unable to add settable data to UDF " + genericUDF.getClass()); + } + } + return newInstance; } /** @@ -1270,6 +1491,7 @@ private static boolean isOpCast(ExprNodeDesc desc) { udfClass == UDFToDouble.class || udfClass == UDFToFloat.class || udfClass == UDFToInteger.class || udfClass == UDFToLong.class || udfClass == UDFToShort.class || udfClass == UDFToString.class || + udfClass == GenericUDFToVarchar.class || udfClass == GenericUDFTimestamp.class || udfClass == GenericUDFToBinary.class || udfClass == GenericUDFToDate.class; } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/SettableUDF.java ql/src/java/org/apache/hadoop/hive/ql/exec/SettableUDF.java new file mode 100644 index 0000000..836e58d --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/SettableUDF.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.exec; + +/** + * Interface to allow passing of parameters to the UDF, before it is initialized. + * For example, to be able to pass the char length parameters to a char type cast. + */ +public interface SettableUDF { + + /** + * Add data to UDF prior to initialization. + * An exception may be thrown if the UDF doesn't know what to do with this data. + * @param params UDF-specific data to add to the UDF + */ + void setParams(Object params) throws UDFArgumentException; + + Object getParams(); + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java index 18a4b02..fda5f22 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.ParameterizedPrimitiveTypeUtils; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; @@ -138,7 +139,9 @@ public static StructObjectInspector getVCSObjectInspector(List vc for (VirtualColumn vc : vcs) { names.add(vc.getName()); inspectors.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( - vc.getTypeInfo().getPrimitiveCategory())); + vc.getTypeInfo().getPrimitiveCategory(), + ParameterizedPrimitiveTypeUtils.getTypeParamsFromTypeInfo( + (PrimitiveTypeInfo) vc.getTypeInfo()))); } return ObjectInspectorFactory.getStandardStructObjectInspector(names, inspectors); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index fa2c1e2..a7cc4b9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -629,7 +629,7 @@ protected static String getTypeStringFromAST(ASTNode typeNode) case HiveParser.TOK_UNIONTYPE: return getUnionTypeStringFromAST(typeNode); default: - return DDLSemanticAnalyzer.getTypeName(typeNode.getType()); + return DDLSemanticAnalyzer.getTypeName(typeNode); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index 932ec66..3002c13 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -38,6 +38,7 @@ import org.antlr.runtime.tree.CommonTree; import org.antlr.runtime.tree.Tree; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; @@ -148,6 +149,7 @@ TokenToTypeName.put(HiveParser.TOK_FLOAT, serdeConstants.FLOAT_TYPE_NAME); TokenToTypeName.put(HiveParser.TOK_DOUBLE, serdeConstants.DOUBLE_TYPE_NAME); TokenToTypeName.put(HiveParser.TOK_STRING, serdeConstants.STRING_TYPE_NAME); + TokenToTypeName.put(HiveParser.TOK_VARCHAR, serdeConstants.VARCHAR_TYPE_NAME); TokenToTypeName.put(HiveParser.TOK_BINARY, serdeConstants.BINARY_TYPE_NAME); TokenToTypeName.put(HiveParser.TOK_DATE, serdeConstants.DATE_TYPE_NAME); TokenToTypeName.put(HiveParser.TOK_DATETIME, serdeConstants.DATETIME_TYPE_NAME); @@ -155,12 +157,54 @@ TokenToTypeName.put(HiveParser.TOK_DECIMAL, serdeConstants.DECIMAL_TYPE_NAME); } - public static String getTypeName(int token) throws SemanticException { + public static String addParamsToTypeName(String typeName, List params) { + StringBuilder sb = new StringBuilder(); + sb.append(typeName); + sb.append("("); + sb.append(StringUtils.join(params, ",")); + sb.append(")"); + return sb.toString(); + } + + public static List getCharParams(String typeName, ASTNode node) + throws SemanticException { + if (node.getChildCount() != 1) { + throw new SemanticException("Bad params for type " + typeName); + } + try { + String lengthStr = node.getChild(0).getText(); + Integer length = Integer.valueOf(lengthStr); + if (length.intValue() < 1) { + throw new SemanticException(typeName + " length must be a positive number"); + } + List paramList = new LinkedList(); + paramList.add(length.toString()); + return paramList; + } catch (NumberFormatException err) { + throw new SemanticException("Bad params for type " + typeName); + } + } + + public static String getTypeName(ASTNode node) throws SemanticException { + int token = node.getType(); + String typeName; + // datetime type isn't currently supported if (token == HiveParser.TOK_DATETIME) { throw new SemanticException(ErrorMsg.UNSUPPORTED_TYPE.getMsg()); } - return TokenToTypeName.get(token); + + switch (token) { + case HiveParser.TOK_VARCHAR: + typeName = TokenToTypeName.get(token); + // Need to validate params here + List paramList = getCharParams(typeName, node); + typeName = addParamsToTypeName(typeName, paramList); + break; + default: + typeName = TokenToTypeName.get(token); + } + return typeName; } static class TablePartition { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g index ff65c8f..ca667d4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g @@ -105,6 +105,7 @@ KW_DATETIME: 'DATETIME'; KW_TIMESTAMP: 'TIMESTAMP'; KW_DECIMAL: 'DECIMAL'; KW_STRING: 'STRING'; +KW_VARCHAR: 'VARCHAR'; KW_ARRAY: 'ARRAY'; KW_STRUCT: 'STRUCT'; KW_MAP: 'MAP'; diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 36d62a6..b7b18da 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -110,6 +110,7 @@ TOK_DATELITERAL; TOK_DATETIME; TOK_TIMESTAMP; TOK_STRING; +TOK_VARCHAR; TOK_BINARY; TOK_DECIMAL; TOK_LIST; @@ -1771,6 +1772,7 @@ primitiveType | KW_STRING -> TOK_STRING | KW_BINARY -> TOK_BINARY | KW_DECIMAL -> TOK_DECIMAL + | KW_VARCHAR LPAREN length=Number RPAREN -> ^(TOK_VARCHAR $length) ; listType diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 1d6e6fe..550ca2f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -168,6 +168,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.BaseTypeParams; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; @@ -5318,6 +5320,41 @@ private Operator genFileSinkPlan(String dest, QB qb, Operator input) } /** + * @param column column expression to convert + * @param tableFieldTypeInfo TypeInfo to convert to + * @return Expression converting column to the type specified by tableFieldTypeInfo + */ + ExprNodeDesc createConversionCast(ExprNodeDesc column, PrimitiveTypeInfo tableFieldTypeInfo) + throws SemanticException { + ExprNodeDesc ret; + + // Get base type, since type string may be parameterized + String baseType = TypeInfoUtils.getBaseName(tableFieldTypeInfo.getTypeName()); + BaseTypeParams typeParams = null; + // If TypeInfo is parameterized, provide the params to the UDF factory method. + typeParams = tableFieldTypeInfo.getParameters(); + if (typeParams != null) { + switch (tableFieldTypeInfo.getPrimitiveCategory()) { + case VARCHAR: + // Nothing to do here - the parameter will be passed to the UDF factory method below + break; + default: + throw new SemanticException("Type cast for " + tableFieldTypeInfo.getPrimitiveCategory() + + " does not take type parameters"); + } + } + + // If the type cast UDF is for a parameterized type, then it should implement + // the SettableUDF interface so that we can pass in the params. + // Not sure if this is the cleanest solution, but there does need to be a way + // to provide the type params to the type cast. + ret = TypeCheckProcFactory.DefaultExprProcessor + .getFuncExprNodeDescWithUdfData(baseType, typeParams, column); + + return ret; + } + + /** * Generate the conversion SelectOperator that converts the columns into the * types that are expected by the table_desc. */ @@ -5388,9 +5425,7 @@ Operator genConversionSelectOperator(String dest, QB qb, Operator input, // cannot convert to complex types column = null; } else { - column = TypeCheckProcFactory.DefaultExprProcessor - .getFuncExprNodeDesc(tableFieldTypeInfo.getTypeName(), - column); + column = createConversionCast(column, (PrimitiveTypeInfo)tableFieldTypeInfo); } if (column == null) { String reason = "Cannot convert column " + i + " from " @@ -5632,9 +5667,7 @@ private Operator genLimitMapRedPlan(String dest, QB qb, Operator input, // cannot convert to complex types column = null; } else { - column = TypeCheckProcFactory.DefaultExprProcessor - .getFuncExprNodeDesc(tableFieldTypeInfo.getTypeName(), - column); + column = createConversionCast(column, (PrimitiveTypeInfo)tableFieldTypeInfo); } if (column == null) { String reason = "Cannot convert column " + posn + " from " @@ -6211,11 +6244,12 @@ private void genJoinOperatorTypeCheck(Operator left, Operator[] right) } // Add implicit type conversion if necessary for (int i = 0; i < right.length; i++) { - if (!commonType.equals(keys.get(i).get(k).getTypeInfo())) { + if (FunctionRegistry.isConversionRequiredForComparison( + keys.get(i).get(k).getTypeInfo(), + commonType)) { keys.get(i).set( k, - TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc( - commonType.getTypeName(), keys.get(i).get(k))); + createConversionCast(keys.get(i).get(k), (PrimitiveTypeInfo)commonType)); } } } @@ -7546,12 +7580,12 @@ private Operator genUnionPlan(String unionalias, String leftalias, * @param unionalias * The alias of the union. * @return - * @throws UDFArgumentException + * @throws SemanticException */ private Operator genInputSelectForUnion( Operator origInputOp, Map origInputFieldMap, String origInputAlias, RowResolver unionoutRR, String unionalias) - throws UDFArgumentException { + throws SemanticException { List columns = new ArrayList(); boolean needsCast = false; @@ -7562,8 +7596,7 @@ private Operator genUnionPlan(String unionalias, String leftalias, lInfo.getTabAlias(), lInfo.getIsVirtualCol(), lInfo.isSkewedCol()); if (!lInfo.getType().equals(unionEntry.getValue().getType())) { needsCast = true; - column = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc( - unionEntry.getValue().getType().getTypeName(), column); + column = createConversionCast(column, (PrimitiveTypeInfo)unionEntry.getValue().getType()); } columns.add(column); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java index 048824a..b6cc04c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.FunctionInfo; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; +import org.apache.hadoop.hive.ql.exec.SettableUDF; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; @@ -61,6 +62,7 @@ import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeParams; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; @@ -561,6 +563,8 @@ public static ColumnExprProcessor getColumnExprProcessor() { serdeConstants.DOUBLE_TYPE_NAME); conversionFunctionTextHashMap.put(HiveParser.TOK_STRING, serdeConstants.STRING_TYPE_NAME); + conversionFunctionTextHashMap.put(HiveParser.TOK_VARCHAR, + serdeConstants.VARCHAR_TYPE_NAME); conversionFunctionTextHashMap.put(HiveParser.TOK_BINARY, serdeConstants.BINARY_TYPE_NAME); conversionFunctionTextHashMap.put(HiveParser.TOK_DATE, @@ -643,7 +647,7 @@ public static String getFunctionText(ASTNode expr, boolean isFunction) { * * @throws UDFArgumentException */ - public static ExprNodeDesc getFuncExprNodeDesc(String udfName, + public static ExprNodeDesc getFuncExprNodeDescWithUdfData(String udfName, Object udfData, ExprNodeDesc... children) throws UDFArgumentException { FunctionInfo fi = FunctionRegistry.getFunctionInfo(udfName); @@ -657,11 +661,23 @@ public static ExprNodeDesc getFuncExprNodeDesc(String udfName, + " is an aggregation function or a table function."); } + // Add udfData to UDF if necessary + if (udfData != null) { + if (genericUDF instanceof SettableUDF) { + ((SettableUDF)genericUDF).setParams(udfData); + } + } + List childrenList = new ArrayList(children.length); childrenList.addAll(Arrays.asList(children)); return ExprNodeGenericFuncDesc.newInstance(genericUDF, childrenList); } + public static ExprNodeDesc getFuncExprNodeDesc(String udfName, + ExprNodeDesc... children) throws UDFArgumentException { + return getFuncExprNodeDescWithUdfData(udfName, null, children); + } + static ExprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr, boolean isFunction, ArrayList children, TypeCheckCtx ctx) throws SemanticException, UDFArgumentException { @@ -758,11 +774,32 @@ static ExprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr, } } + // getGenericUDF() actually clones the UDF. Just call it once and reuse. + GenericUDF genericUDF = fi.getGenericUDF(); + if (!fi.isNative()) { ctx.getUnparseTranslator().addIdentifierTranslation( (ASTNode) expr.getChild(0)); } + // Handle type casts that may contain type parameters + if (isFunction) { + ASTNode funcNameNode = (ASTNode)expr.getChild(0); + switch (funcNameNode.getType()) { + case HiveParser.TOK_VARCHAR: + // Add type params + VarcharTypeParams varcharTypeParams = new VarcharTypeParams(); + varcharTypeParams.length = Integer.valueOf((funcNameNode.getChild(0).getText())); + if (genericUDF != null) { + ((SettableUDF)genericUDF).setParams(varcharTypeParams); + } + break; + default: + // Do nothing + break; + } + } + // Detect UDTF's in nested SELECT, GROUP BY, etc as they aren't // supported if (fi.getGenericUDTF() != null) { @@ -777,8 +814,8 @@ static ExprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr, throw new SemanticException(ErrorMsg.UDAF_INVALID_LOCATION.getMsg(expr)); } } - if (!ctx.getAllowStatefulFunctions() && (fi.getGenericUDF() != null)) { - if (FunctionRegistry.isStateful(fi.getGenericUDF())) { + if (!ctx.getAllowStatefulFunctions() && (genericUDF != null)) { + if (FunctionRegistry.isStateful(genericUDF)) { throw new SemanticException( ErrorMsg.UDF_STATEFUL_INVALID_LOCATION.getMsg()); } @@ -786,7 +823,7 @@ static ExprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr, // Try to infer the type of the constant only if there are two // nodes, one of them is column and the other is numeric const - if (fi.getGenericUDF() instanceof GenericUDFBaseCompare + if (genericUDF instanceof GenericUDFBaseCompare && children.size() == 2 && ((children.get(0) instanceof ExprNodeConstantDesc && children.get(1) instanceof ExprNodeColumnDesc) @@ -843,7 +880,7 @@ static ExprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr, // however, if we already tried this, or the column is NUMBER type and // the operator is EQUAL, return false due to the type mismatch if (triedDouble || - (fi.getGenericUDF() instanceof GenericUDFOPEqual + (genericUDF instanceof GenericUDFOPEqual && !columnType.equals(serdeConstants.STRING_TYPE_NAME))) { return new ExprNodeConstantDesc(false); } @@ -861,7 +898,7 @@ static ExprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr, } } - desc = ExprNodeGenericFuncDesc.newInstance(fi.getGenericUDF(), children); + desc = ExprNodeGenericFuncDesc.newInstance(genericUDF, children); } // UDFOPPositive is a no-op. // However, we still create it, and then remove it here, to make sure we diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java index 25d04e1..628dab9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java @@ -25,10 +25,13 @@ import java.util.Map; import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils; import org.apache.hadoop.hive.ql.io.HiveOutputFormat; @@ -36,7 +39,8 @@ import org.apache.hadoop.hive.ql.parse.ParseUtils; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.serde2.SerDeUtils; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; /** * CreateTableDesc. @@ -45,6 +49,7 @@ @Explain(displayName = "Create Table") public class CreateTableDesc extends DDLDesc implements Serializable { private static final long serialVersionUID = 1L; + private static Log LOG = LogFactory.getLog(CreateTableDesc.class); String databaseName; String tableName; boolean isExternal; @@ -461,10 +466,13 @@ public void validate() while (partColsIter.hasNext()) { FieldSchema fs = partColsIter.next(); String partCol = fs.getName(); - PrimitiveObjectInspectorUtils.PrimitiveTypeEntry pte = PrimitiveObjectInspectorUtils - .getTypeEntryFromTypeName( - fs.getType()); - if(null == pte){ + TypeInfo pti = null; + try { + pti = TypeInfoFactory.getPrimitiveTypeInfo(fs.getType()); + } catch (Exception err) { + LOG.error(err); + } + if(null == pti){ throw new SemanticException(ErrorMsg.PARTITION_COLUMN_NON_PRIMITIVE.getMsg() + " Found " + partCol + " of type: " + fs.getType()); } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/GenericUDFEncode.java ql/src/java/org/apache/hadoop/hive/ql/udf/GenericUDFEncode.java index cb519e5..339bb98 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/GenericUDFEncode.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/GenericUDFEncode.java @@ -19,6 +19,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping; import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Text; @@ -30,8 +32,8 @@ "is null, the result will also be null") public class GenericUDFEncode extends GenericUDF { private transient CharsetEncoder encoder = null; - private transient StringObjectInspector stringOI = null; - private transient StringObjectInspector charsetOI = null; + private transient PrimitiveObjectInspector stringOI = null; + private transient PrimitiveObjectInspector charsetOI = null; private transient BytesWritable result = new BytesWritable(); @Override @@ -41,23 +43,25 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen } if (arguments[0].getCategory() != Category.PRIMITIVE || - ((PrimitiveObjectInspector)arguments[0]).getPrimitiveCategory() != PrimitiveCategory.STRING){ + PrimitiveGrouping.STRING_GROUP != PrimitiveObjectInspectorUtils.getPrimitiveGrouping( + ((PrimitiveObjectInspector)arguments[0]).getPrimitiveCategory())){ throw new UDFArgumentTypeException(0, "The first argument to Encode() must be a string"); } - stringOI = (StringObjectInspector) arguments[0]; + stringOI = (PrimitiveObjectInspector) arguments[0]; if (arguments[1].getCategory() != Category.PRIMITIVE || - ((PrimitiveObjectInspector)arguments[1]).getPrimitiveCategory() != PrimitiveCategory.STRING){ + PrimitiveGrouping.STRING_GROUP != PrimitiveObjectInspectorUtils.getPrimitiveGrouping( + ((PrimitiveObjectInspector)arguments[1]).getPrimitiveCategory())){ throw new UDFArgumentTypeException(1, "The second argument to Encode() must be a string"); } - charsetOI = (StringObjectInspector) arguments[1]; + charsetOI = (PrimitiveObjectInspector) arguments[1]; // If the character set for encoding is constant, we can optimize that - StringObjectInspector charSetOI = (StringObjectInspector) arguments[1]; - if (charSetOI instanceof ConstantObjectInspector){ - String charSetName = ((Text) ((ConstantObjectInspector) charSetOI).getWritableConstantValue()).toString(); + if (charsetOI instanceof ConstantObjectInspector){ + String charSetName = + ((ConstantObjectInspector) arguments[1]).getWritableConstantValue().toString(); encoder = Charset.forName(charSetName).newEncoder().onMalformedInput(CodingErrorAction.REPORT).onUnmappableCharacter(CodingErrorAction.REPORT); } @@ -68,7 +72,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen @Override public Object evaluate(DeferredObject[] arguments) throws HiveException { - String value = stringOI.getPrimitiveJavaObject(arguments[0].get()); + String value = PrimitiveObjectInspectorUtils.getString(arguments[0].get(), stringOI); if (value == null) { return null; } @@ -81,7 +85,8 @@ public Object evaluate(DeferredObject[] arguments) throws HiveException { throw new HiveException(e); } } else { - encoded = Charset.forName(charsetOI.getPrimitiveJavaObject(arguments[1].get())).encode(value); + encoded = Charset.forName( + PrimitiveObjectInspectorUtils.getString(arguments[1].get(), charsetOI)).encode(value); } result.setSize(encoded.limit()); encoded.get(result.getBytes(), 0, encoded.limit()); diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java index 17a79f0..6a4272a 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.hive.serde2.ByteStream; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java index 23a3c0c..e254813 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java @@ -83,6 +83,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) case DOUBLE: return new GenericUDAFDoubleStatsEvaluator(); case STRING: + case VARCHAR: return new GenericUDAFStringStatsEvaluator(); case BINARY: return new GenericUDAFBinaryStatsEvaluator(); diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseCompare.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseCompare.java index 2cb65d8..b1dda71 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseCompare.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseCompare.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.udf.generic; +import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; @@ -29,15 +30,18 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeParams; import org.apache.hadoop.io.BooleanWritable; /** @@ -133,36 +137,20 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen TypeInfo oiTypeInfo0 = TypeInfoUtils.getTypeInfoFromObjectInspector(arguments[0]); TypeInfo oiTypeInfo1 = TypeInfoUtils.getTypeInfoFromObjectInspector(arguments[1]); - if (oiTypeInfo0 != oiTypeInfo1) { + if (oiTypeInfo0 == oiTypeInfo1 + || TypeInfoUtils.doPrimitiveCategoriesMatch(oiTypeInfo0, oiTypeInfo1)) { + compareType = CompareType.SAME_TYPE; + } else { compareType = CompareType.NEED_CONVERT; + TypeInfo compareType = FunctionRegistry.getCommonClassForComparison(oiTypeInfo0, oiTypeInfo1); - if ((oiTypeInfo0.equals(TypeInfoFactory.stringTypeInfo) - && oiTypeInfo1.equals(TypeInfoFactory.dateTypeInfo)) - || (oiTypeInfo0.equals(TypeInfoFactory.dateTypeInfo) - && oiTypeInfo1.equals(TypeInfoFactory.stringTypeInfo))) { - // Date should be comparable with string - compareOI = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo( - TypeInfoFactory.stringTypeInfo); - - } else if (oiTypeInfo0.equals(TypeInfoFactory.stringTypeInfo) - || oiTypeInfo1.equals(TypeInfoFactory.stringTypeInfo)) { - // If either argument is a string, we convert to a double because a number - // in string form should always be convertible into a double - compareOI = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo( - TypeInfoFactory.doubleTypeInfo); - } else { - TypeInfo compareType = FunctionRegistry.getCommonClass(oiTypeInfo0, oiTypeInfo1); - - // For now, we always convert to double if we can't find a common type - compareOI = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo( - (compareType == null) ? - TypeInfoFactory.doubleTypeInfo : compareType); - } + // For now, we always convert to double if we can't find a common type + compareOI = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo( + (compareType == null) ? + TypeInfoFactory.doubleTypeInfo : compareType); converter0 = ObjectInspectorConverters.getConverter(arguments[0], compareOI); converter1 = ObjectInspectorConverters.getConverter(arguments[1], compareOI); - } else { - compareType = CompareType.SAME_TYPE; } } return PrimitiveObjectInspectorFactory.writableBooleanObjectInspector; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFConcat.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFConcat.java new file mode 100644 index 0000000..c975383 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFConcat.java @@ -0,0 +1,192 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.StringConverter; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeParams; +import org.apache.hadoop.io.BytesWritable; + +/** + * GenericUDFConcat. + */ +@Description(name = "concat", +value = "_FUNC_(str1, str2, ... strN) - returns the concatenation of str1, str2, ... strN or "+ + "_FUNC_(bin1, bin2, ... binN) - returns the concatenation of bytes in binary data " + + " bin1, bin2, ... binN", +extended = "Returns NULL if any argument is NULL.\n" ++ "Example:\n" ++ " > SELECT _FUNC_('abc', 'def') FROM src LIMIT 1;\n" ++ " 'abcdef'") +public class GenericUDFConcat extends GenericUDF { + private ObjectInspector[] argumentOIs; + private StringConverter[] stringConverters; + private PrimitiveCategory returnType = PrimitiveCategory.STRING; + private BytesWritable[] bw; + private GenericUDFUtils.StringHelper returnHelper; + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + + // Loop through all the inputs to determine the appropriate return type/length. + // Either all arguments are binary, or all columns are non-binary. + // Return type: + // All VARCHAR inputs: return VARCHAR + // All BINARY inputs: return BINARY + // Otherwise return STRING + argumentOIs = arguments; + + PrimitiveCategory currentCategory; + PrimitiveObjectInspector poi; + boolean fixedLengthReturnValue = true; + int returnLength = 0; // Only for char/varchar return types + for (int idx = 0; idx < arguments.length; ++idx) { + if (arguments[idx].getCategory() != Category.PRIMITIVE) { + throw new UDFArgumentException("CONCAT only takes primitive arguments"); + } + poi = (PrimitiveObjectInspector)arguments[idx]; + currentCategory = poi.getPrimitiveCategory(); + if (idx == 0) { + returnType = currentCategory; + } + switch (currentCategory) { + case BINARY: + fixedLengthReturnValue = false; + if (returnType != currentCategory) { + throw new UDFArgumentException( + "CONCAT cannot take a mix of binary and non-binary arguments"); + } + break; + case VARCHAR: + if (returnType == PrimitiveCategory.BINARY) { + throw new UDFArgumentException( + "CONCAT cannot take a mix of binary and non-binary arguments"); + } + break; + default: + if (returnType == PrimitiveCategory.BINARY) { + throw new UDFArgumentException( + "CONCAT cannot take a mix of binary and non-binary arguments"); + } + returnType = PrimitiveCategory.STRING; + fixedLengthReturnValue = false; + break; + } + if (fixedLengthReturnValue) { + returnLength += GenericUDFUtils.StringHelper.getFixedStringSizeForType(poi); + } + } + + if (returnType == PrimitiveCategory.BINARY) { + bw = new BytesWritable[arguments.length]; + return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector; + } else { + // treat all inputs as string, the return value will be converted to the appropriate type. + createStringConverters(); + returnHelper = new GenericUDFUtils.StringHelper(returnType); + switch (returnType) { + case STRING: + return PrimitiveObjectInspectorFactory.writableStringObjectInspector; + case VARCHAR: + VarcharTypeParams varcharParams = new VarcharTypeParams(); + varcharParams.setLength(returnLength); + return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( + returnType, varcharParams); + default: + throw new UDFArgumentException("Unexpected CONCAT return type of " + returnType); + } + } + } + + private void createStringConverters() { + stringConverters = new StringConverter[argumentOIs.length]; + for (int idx = 0; idx < argumentOIs.length; ++idx) { + stringConverters[idx] = new StringConverter((PrimitiveObjectInspector) argumentOIs[idx]); + } + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + if (returnType == PrimitiveCategory.BINARY) { + return binaryEvaluate(arguments); + } else { + return returnHelper.setReturnValue(stringEvaluate(arguments)); + } + } + + public Object binaryEvaluate(DeferredObject[] arguments) throws HiveException { + int len = 0; + for (int idx = 0; idx < arguments.length; ++idx) { + bw[idx] = ((BinaryObjectInspector)argumentOIs[idx]) + .getPrimitiveWritableObject(arguments[idx].get()); + if (bw[idx] == null){ + return null; + } + len += bw[idx].getLength(); + } + + byte[] out = new byte[len]; + int curLen = 0; + // Need to iterate twice since BytesWritable doesn't support append. + for (BytesWritable bytes : bw){ + System.arraycopy(bytes.getBytes(), 0, out, curLen, bytes.getLength()); + curLen += bytes.getLength(); + } + return new BytesWritable(out); + } + + public String stringEvaluate(DeferredObject[] arguments) throws HiveException { + StringBuilder sb = new StringBuilder(); + for (int idx = 0; idx < arguments.length; ++idx) { + String val = null; + if (arguments[idx] != null) { + val = (String) stringConverters[idx].convert(arguments[idx].get()); + } + if (val == null) { + return null; + } + sb.append(val); + } + return sb.toString(); + } + + @Override + public String getDisplayString(String[] children) { + StringBuilder sb = new StringBuilder(); + sb.append("concat("); + if (children.length > 0) { + sb.append(children[0]); + for (int i = 1; i < children.length; i++) { + sb.append(", "); + sb.append(children[i]); + } + } + sb.append(")"); + return sb.toString(); + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFConcatWS.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFConcatWS.java index feb9215..8763942 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFConcatWS.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFConcatWS.java @@ -27,7 +27,11 @@ import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping; import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; import org.apache.hadoop.io.Text; @@ -61,15 +65,12 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen for (int i = 0; i < arguments.length; i++) { switch(arguments[i].getCategory()) { case LIST: - if (((ListObjectInspector)arguments[i]).getListElementObjectInspector() - .getTypeName().equals(serdeConstants.STRING_TYPE_NAME) - || ((ListObjectInspector)arguments[i]).getListElementObjectInspector() - .getTypeName().equals(serdeConstants.VOID_TYPE_NAME)) { - break; + if (isStringOrVoidType( + ((ListObjectInspector) arguments[i]).getListElementObjectInspector())) { + break; } case PRIMITIVE: - if (arguments[i].getTypeName().equals(serdeConstants.STRING_TYPE_NAME) - || arguments[i].getTypeName().equals(serdeConstants.VOID_TYPE_NAME)) { + if (isStringOrVoidType(arguments[i])) { break; } default: @@ -84,6 +85,18 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen return PrimitiveObjectInspectorFactory.writableStringObjectInspector; } + protected boolean isStringOrVoidType(ObjectInspector oi) { + if (oi.getCategory() == Category.PRIMITIVE) { + if (PrimitiveGrouping.STRING_GROUP + == PrimitiveObjectInspectorUtils.getPrimitiveGrouping( + ((PrimitiveObjectInspector) oi).getPrimitiveCategory()) + || ((PrimitiveObjectInspector) oi).getPrimitiveCategory() == PrimitiveCategory.VOID) { + return true; + } + } + return false; + } + private final Text resultText = new Text(); @Override @@ -91,8 +104,8 @@ public Object evaluate(DeferredObject[] arguments) throws HiveException { if (arguments[0].get() == null) { return null; } - String separator = ((StringObjectInspector) argumentOIs[0]) - .getPrimitiveJavaObject(arguments[0].get()); + String separator = PrimitiveObjectInspectorUtils.getString( + arguments[0].get(), (PrimitiveObjectInspector)argumentOIs[0]); StringBuilder sb = new StringBuilder(); boolean first = true; @@ -116,8 +129,8 @@ public Object evaluate(DeferredObject[] arguments) throws HiveException { sb.append(strArrayOI.getListElement(strArray, j)); } } else { - sb.append(((StringObjectInspector) argumentOIs[i]) - .getPrimitiveJavaObject(arguments[i].get())); + sb.append(PrimitiveObjectInspectorUtils.getString( + arguments[i].get(), (PrimitiveObjectInspector)argumentOIs[i])); } } } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLower.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLower.java new file mode 100644 index 0000000..b3b8046 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLower.java @@ -0,0 +1,110 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.StringConverter; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeParams; + +/** + * UDFLower. + * + */ +@Description(name = "lower,lcase", +value = "_FUNC_(str) - Returns str with all characters changed to lowercase", +extended = "Example:\n" ++ " > SELECT _FUNC_('Facebook') FROM src LIMIT 1;\n" + " 'facebook'") +public class GenericUDFLower extends GenericUDF { + private PrimitiveObjectInspector argumentOI; + private StringConverter stringConverter; + private PrimitiveCategory returnType = PrimitiveCategory.STRING; + private GenericUDFUtils.StringHelper returnHelper; + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + if (arguments.length < 0) { + throw new UDFArgumentLengthException( + "LOWER requires 1 argument, got " + arguments.length); + } + + if (arguments[0].getCategory() != Category.PRIMITIVE) { + throw new UDFArgumentException( + "LOWER only takes primitive types, got " + argumentOI.getTypeName()); + } + argumentOI = (PrimitiveObjectInspector) arguments[0]; + + stringConverter = new PrimitiveObjectInspectorConverter.StringConverter(argumentOI); + PrimitiveCategory inputType = argumentOI.getPrimitiveCategory(); + ObjectInspector outputOI = null; + switch (inputType) { + case VARCHAR: + // return type should have same length as the input. + returnType = inputType; + VarcharTypeParams varcharParams = new VarcharTypeParams(); + varcharParams.setLength(GenericUDFUtils.StringHelper.getFixedStringSizeForType(argumentOI)); + outputOI = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( + returnType, varcharParams); + break; + default: + returnType = PrimitiveCategory.STRING; + outputOI = PrimitiveObjectInspectorFactory.writableStringObjectInspector; + break; + } + returnHelper = new GenericUDFUtils.StringHelper(returnType); + return outputOI; + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + String val = null; + if (arguments[0] != null) { + val = (String) stringConverter.convert(arguments[0].get()); + } + if (val == null) { + return null; + } + val = val.toLowerCase(); + return returnHelper.setReturnValue(val); + } + + @Override + public String getDisplayString(String[] children) { + StringBuilder sb = new StringBuilder(); + sb.append("lower("); + if (children.length > 0) { + sb.append(children[0]); + for (int i = 1; i < children.length; i++) { + sb.append(","); + sb.append(children[i]); + } + } + sb.append(")"); + return sb.toString(); + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect2.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect2.java index 033ee1f..27369c5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect2.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect2.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.BytesWritable; @@ -93,8 +94,12 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen try { method = findMethod(targetClass, methodName.toString(), null, true); + // While getTypeFor() returns a TypeEntry, we won't actually be able to get any + // type parameter information from this since the TypeEntry is derived from a return type. + PrimitiveTypeEntry typeEntry = getTypeFor(method.getReturnType()); returnOI = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( - getTypeFor(method.getReturnType()).primitiveCategory); + typeEntry.primitiveCategory, + typeEntry.typeParams); returnObj = (Writable) returnOI.getPrimitiveWritableClass().newInstance(); } catch (Exception e) { throw new UDFArgumentException(e); diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java index c24a50d..4695cd5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java @@ -24,9 +24,15 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; @@ -41,30 +47,30 @@ + " delimiters are used: ',' as delimiter1 and '=' as delimiter2.") public class GenericUDFStringToMap extends GenericUDF { private final HashMap ret = new HashMap(); - private transient StringObjectInspector soi_text, soi_de1 = null, soi_de2 = null; + private transient Converter soi_text, soi_de1 = null, soi_de2 = null; final static String default_de1 = ","; final static String default_de2 = ":"; @Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { - if (!TypeInfoUtils.getTypeInfoFromObjectInspector(arguments[0]).equals( - TypeInfoFactory.stringTypeInfo) - || (arguments.length > 1 && - !TypeInfoUtils.getTypeInfoFromObjectInspector(arguments[1]).equals( - TypeInfoFactory.stringTypeInfo)) - || (arguments.length > 2 && - !TypeInfoUtils.getTypeInfoFromObjectInspector(arguments[2]).equals( - TypeInfoFactory.stringTypeInfo))) { - throw new UDFArgumentException("All argument should be string"); + for (int idx = 0; idx < Math.min(arguments.length, 3); ++idx) { + if (arguments[idx].getCategory() != Category.PRIMITIVE + || PrimitiveObjectInspectorUtils.getPrimitiveGrouping( + ((PrimitiveObjectInspector) arguments[idx]).getPrimitiveCategory()) + != PrimitiveGrouping.STRING_GROUP) { + throw new UDFArgumentException("All argument should be string/character type"); + } } - - soi_text = (StringObjectInspector) arguments[0]; + soi_text = ObjectInspectorConverters.getConverter(arguments[0], + PrimitiveObjectInspectorFactory.javaStringObjectInspector); if (arguments.length > 1) { - soi_de1 = (StringObjectInspector) arguments[1]; + soi_de1 = ObjectInspectorConverters.getConverter(arguments[1], + PrimitiveObjectInspectorFactory.javaStringObjectInspector); } if (arguments.length > 2) { - soi_de2 = (StringObjectInspector) arguments[2]; + soi_de2 = ObjectInspectorConverters.getConverter(arguments[2], + PrimitiveObjectInspectorFactory.javaStringObjectInspector); } return ObjectInspectorFactory.getStandardMapObjectInspector( @@ -75,11 +81,11 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen @Override public Object evaluate(DeferredObject[] arguments) throws HiveException { ret.clear(); - String text = soi_text.getPrimitiveJavaObject(arguments[0].get()); + String text = (String) soi_text.convert(arguments[0].get()); String delimiter1 = (soi_de1 == null) ? - default_de1 : soi_de1.getPrimitiveJavaObject(arguments[1].get()); + default_de1 : (String) soi_de1.convert(arguments[1].get()); String delimiter2 = (soi_de2 == null) ? - default_de2 : soi_de2.getPrimitiveJavaObject(arguments[2].get()); + default_de2 : (String) soi_de2.convert(arguments[2].get()); String[] keyValuePairs = text.split(delimiter1); diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java index 099e3b0..9188c38 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java @@ -25,6 +25,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.DateConverter; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping; /** * GenericUDFToDate @@ -49,10 +51,11 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen } try { argumentOI = (PrimitiveObjectInspector) arguments[0]; - switch (argumentOI.getPrimitiveCategory()) { - case DATE: - case STRING: - case TIMESTAMP: + PrimitiveGrouping pg = + PrimitiveObjectInspectorUtils.getPrimitiveGrouping(argumentOI.getPrimitiveCategory()); + switch (pg) { + case DATE_GROUP: + case STRING_GROUP: break; default: throw new UDFArgumentException( diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToVarchar.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToVarchar.java new file mode 100644 index 0000000..a48df71 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToVarchar.java @@ -0,0 +1,121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.udf.generic; + +import java.io.Serializable; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.ql.exec.SettableUDF; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.HiveVarcharConverter; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveVarcharObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeParams; + + +public class GenericUDFToVarchar extends GenericUDF + implements SettableUDF, Serializable { + private static final Log LOG = LogFactory.getLog(GenericUDFToVarchar.class.getName()); + private PrimitiveObjectInspector argumentOI; + private HiveVarcharConverter converter; + private VarcharTypeParams typeParams; + + public GenericUDFToVarchar() { + LOG.info("testing"); + } + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + if (arguments.length != 1) { + throw new UDFArgumentException("VARCHAR cast requires a value argument"); + } + try { + argumentOI = (PrimitiveObjectInspector) arguments[0]; + } catch (ClassCastException e) { + throw new UDFArgumentException( + "The function VARCHAR takes only primitive types"); + } + + // Check if this UDF has been provided with type params for the output varchar type + SettableHiveVarcharObjectInspector outputOI; + if (typeParams != null) { + outputOI = (SettableHiveVarcharObjectInspector) + PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( + PrimitiveCategory.VARCHAR, typeParams); + } else { + outputOI = PrimitiveObjectInspectorFactory.writableHiveVarcharObjectInspector; + } + + converter = new HiveVarcharConverter(argumentOI, outputOI); + return outputOI; + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + Object o0 = arguments[0].get(); + if (o0 == null) { + return null; + } + + return converter.convert(o0); + } + + @Override + public String getDisplayString(String[] children) { + assert (children.length == 1); + StringBuilder sb = new StringBuilder(); + sb.append("CAST( "); + sb.append(children[0]); + sb.append(" AS VARCHAR("); + String paramsStr = ""; + if (typeParams != null) { + paramsStr = typeParams.toString(); + } + sb.append(paramsStr); + sb.append(")"); + return sb.toString(); + } + +/** + * Provide varchar type parameters for the output object inspector. + * This should be done before the UDF is initialized. + */ + @Override + public void setParams(Object typeParams) throws UDFArgumentException { + if (converter != null) { + LOG.warn("Type converter already initialized, setting type params now will not be useful"); + } + if (typeParams instanceof VarcharTypeParams) { + this.typeParams = (VarcharTypeParams)typeParams; + } else { + throw new UDFArgumentException( + "Was expecting VarcharTypeParams, instead got " + typeParams.getClass().getName()); + } + } + + @Override + public Object getParams() { + return typeParams; + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUpper.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUpper.java new file mode 100644 index 0000000..dbe2934 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUpper.java @@ -0,0 +1,110 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.StringConverter; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeParams; + +/** + * UDFUpper. + * + */ +@Description(name = "upper,ucase", + value = "_FUNC_(str) - Returns str with all characters changed to uppercase", + extended = "Example:\n" + + " > SELECT _FUNC_('Facebook') FROM src LIMIT 1;\n" + " 'FACEBOOK'") +public class GenericUDFUpper extends GenericUDF { + private PrimitiveObjectInspector argumentOI; + private StringConverter stringConverter; + private PrimitiveCategory returnType = PrimitiveCategory.STRING; + private GenericUDFUtils.StringHelper returnHelper; + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + if (arguments.length < 0) { + throw new UDFArgumentLengthException( + "UPPER requires 1 argument, got " + arguments.length); + } + + if (arguments[0].getCategory() != Category.PRIMITIVE) { + throw new UDFArgumentException( + "UPPER only takes primitive types, got " + argumentOI.getTypeName()); + } + argumentOI = (PrimitiveObjectInspector) arguments[0]; + + stringConverter = new PrimitiveObjectInspectorConverter.StringConverter(argumentOI); + PrimitiveCategory inputType = argumentOI.getPrimitiveCategory(); + ObjectInspector outputOI = null; + switch (inputType) { + case VARCHAR: + // return type should have same length as the input. + returnType = inputType; + VarcharTypeParams varcharParams = new VarcharTypeParams(); + varcharParams.setLength(GenericUDFUtils.StringHelper.getFixedStringSizeForType(argumentOI)); + outputOI = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( + returnType, varcharParams); + break; + default: + returnType = PrimitiveCategory.STRING; + outputOI = PrimitiveObjectInspectorFactory.writableStringObjectInspector; + break; + } + returnHelper = new GenericUDFUtils.StringHelper(returnType); + return outputOI; + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + String val = null; + if (arguments[0] != null) { + val = (String) stringConverter.convert(arguments[0].get()); + } + if (val == null) { + return null; + } + val = val.toUpperCase(); + return returnHelper.setReturnValue(val); + } + + @Override + public String getDisplayString(String[] children) { + StringBuilder sb = new StringBuilder(); + sb.append("upper("); + if (children.length > 0) { + sb.append(children[0]); + for (int i = 1; i < children.length; i++) { + sb.append(","); + sb.append(children[i]); + } + } + sb.append(")"); + return sb.toString(); + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java index d8052c8..6815195 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java @@ -29,17 +29,21 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.IdentityConverter; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.VoidObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeParams; import org.apache.hadoop.io.Text; /** @@ -350,6 +354,69 @@ public ConversionHelper(Method m, ObjectInspector[] parameterOIs) }; /** + * Helper class for UDFs returning string/varchar/char + */ + public static class StringHelper { + + protected Object returnValue; + protected PrimitiveCategory type; + + public StringHelper(PrimitiveCategory type) throws UDFArgumentException { + this.type = type; + switch (type) { + case STRING: + returnValue = new Text(); + break; + case VARCHAR: + returnValue = new HiveVarcharWritable(); + break; + default: + throw new UDFArgumentException("Unexpected non-string type " + type); + } + } + + public Object setReturnValue(String val) throws UDFArgumentException { + if (val == null) { + return null; + } + switch (type) { + case STRING: + ((Text)returnValue).set(val); + return returnValue; + case VARCHAR: + ((HiveVarcharWritable)returnValue).set(val); + return returnValue; + default: + throw new UDFArgumentException("Bad return type " + type); + } + } + + /** + * Helper function to help GenericUDFs determine the return type + * character length for char/varchar. + * @param poi PrimitiveObjectInspector representing the type + * @return character length of the type + * @throws UDFArgumentException + */ + public static int getFixedStringSizeForType(PrimitiveObjectInspector poi) + throws UDFArgumentException { + // TODO: we can support date, int, .. any types which would have a fixed length value + switch (poi.getPrimitiveCategory()) { + case VARCHAR: + VarcharTypeParams varcharParams = null; + varcharParams = (VarcharTypeParams) poi.getTypeParams(); + if (varcharParams == null || varcharParams.length < 0) { + throw new UDFArgumentException("varchar type used without type params"); + } + return varcharParams.length; + default: + throw new UDFArgumentException("No fixed size for type " + poi.getTypeName()); + } + } + + } + + /** * Return an ordinal from an integer. */ public static String getOrdinal(int i) { diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java index 3875b5d..ee318af 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java @@ -25,10 +25,20 @@ import junit.framework.Assert; import junit.framework.TestCase; +import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.io.BytesWritable; @@ -39,6 +49,7 @@ public class TestUDF { public void same(DoubleWritable x, DoubleWritable y) {} public void same(HiveDecimalWritable x, HiveDecimalWritable y) {} + public void same(Text x, Text y) {} public void one(IntWritable x, HiveDecimalWritable y) {} public void one(IntWritable x, DoubleWritable y) {} public void one(IntWritable x, IntWritable y) {} @@ -47,8 +58,16 @@ public void mismatch(TimestampWritable x, HiveDecimalWritable y) {} public void mismatch(BytesWritable x, DoubleWritable y) {} } + TypeInfo varchar5; + TypeInfo varchar10; + TypeInfo maxVarchar; + @Override protected void setUp() { + String maxVarcharTypeName = "varchar(" + HiveVarchar.MAX_VARCHAR_LENGTH + ")"; + maxVarchar = TypeInfoFactory.getPrimitiveTypeInfo(maxVarcharTypeName); + varchar10 = TypeInfoFactory.getPrimitiveTypeInfo("varchar(10)"); + varchar5 = TypeInfoFactory.getPrimitiveTypeInfo("varchar(5)"); } private void implicit(TypeInfo a, TypeInfo b, boolean convertible) { @@ -62,6 +81,21 @@ public void testImplicitConversion() { implicit(TypeInfoFactory.stringTypeInfo, TypeInfoFactory.decimalTypeInfo, true); implicit(TypeInfoFactory.dateTypeInfo, TypeInfoFactory.decimalTypeInfo, false); implicit(TypeInfoFactory.timestampTypeInfo, TypeInfoFactory.decimalTypeInfo, false); + implicit(varchar10, TypeInfoFactory.stringTypeInfo, true); + implicit(TypeInfoFactory.stringTypeInfo, varchar10, true); + + // Try with parameterized varchar types + TypeInfo varchar10 = TypeInfoFactory.getPrimitiveTypeInfo("varchar(10)"); + TypeInfo varchar20 = TypeInfoFactory.getPrimitiveTypeInfo("varchar(20)"); + + implicit(varchar10, TypeInfoFactory.stringTypeInfo, true); + implicit(varchar20, TypeInfoFactory.stringTypeInfo, true); + implicit(TypeInfoFactory.stringTypeInfo, varchar10, true); + implicit(TypeInfoFactory.stringTypeInfo, varchar20, true); + implicit(varchar20, varchar10, true); + + implicit(TypeInfoFactory.intTypeInfo, varchar10, true); + implicit(TypeInfoFactory.intTypeInfo, TypeInfoFactory.stringTypeInfo, true); } private void verify(Class udf, String name, TypeInfo ta, TypeInfo tb, @@ -80,8 +114,8 @@ private void verify(Class udf, String name, TypeInfo ta, TypeInfo tb, } assert(!throwException); assertEquals(2, result.getParameterTypes().length); - assertEquals(result.getParameterTypes()[0], a); - assertEquals(result.getParameterTypes()[1], b); + assertEquals(a, result.getParameterTypes()[0]); + assertEquals(b, result.getParameterTypes()[1]); } public void testGetMethodInternal() { @@ -110,12 +144,15 @@ public void testGetMethodInternal() { verify(TestUDF.class, "one", TypeInfoFactory.intTypeInfo, TypeInfoFactory.intTypeInfo, IntWritable.class, IntWritable.class, false); + // Passing varchar arguments should prefer the version of evaluate() with Text args. + verify(TestUDF.class, "same", varchar5, varchar10, Text.class, Text.class, false); + verify(TestUDF.class, "mismatch", TypeInfoFactory.voidTypeInfo, TypeInfoFactory.intTypeInfo, null, null, true); } private void common(TypeInfo a, TypeInfo b, TypeInfo result) { - assertEquals(FunctionRegistry.getCommonClass(a,b), result); + assertEquals(result, FunctionRegistry.getCommonClass(a,b)); } public void testCommonClass() { @@ -127,10 +164,13 @@ public void testCommonClass() { TypeInfoFactory.decimalTypeInfo); common(TypeInfoFactory.doubleTypeInfo, TypeInfoFactory.stringTypeInfo, TypeInfoFactory.stringTypeInfo); + + common(TypeInfoFactory.stringTypeInfo, varchar10, TypeInfoFactory.stringTypeInfo); + common(varchar10, TypeInfoFactory.stringTypeInfo, TypeInfoFactory.stringTypeInfo); } private void comparison(TypeInfo a, TypeInfo b, TypeInfo result) { - assertEquals(FunctionRegistry.getCommonClassForComparison(a,b), result); + assertEquals(result, FunctionRegistry.getCommonClassForComparison(a,b)); } public void testCommonClassComparison() { @@ -142,6 +182,95 @@ public void testCommonClassComparison() { TypeInfoFactory.decimalTypeInfo); comparison(TypeInfoFactory.doubleTypeInfo, TypeInfoFactory.stringTypeInfo, TypeInfoFactory.doubleTypeInfo); + + comparison(TypeInfoFactory.dateTypeInfo, TypeInfoFactory.stringTypeInfo, + TypeInfoFactory.stringTypeInfo); + comparison(TypeInfoFactory.stringTypeInfo, TypeInfoFactory.dateTypeInfo, + TypeInfoFactory.stringTypeInfo); + + comparison(TypeInfoFactory.stringTypeInfo, varchar10, TypeInfoFactory.stringTypeInfo); + comparison(varchar10, TypeInfoFactory.stringTypeInfo, TypeInfoFactory.stringTypeInfo); + comparison(varchar5, varchar10, varchar10); + } + + /** + * Method to print out the comparison/conversion behavior for data types. + */ + public void testPrintTypeCompatibility() { + if (true) { + return; + } + + String[] typeStrings = { + "void", "boolean", "tinyint", "smallint", "int", "bigint", "float", "double", + "string", "timestamp", "date", "binary", "decimal", "varchar(10)", "varchar(5)", + }; + for (String cat1 : typeStrings) { + TypeInfo ti1 = null; + try { + ti1 = TypeInfoUtils.getTypeInfoFromTypeString(cat1); + } catch (Exception err) { + System.out.println(err); + System.out.println("Unable to get TypeInfo for " + cat1 + ", skipping ..."); + continue; + } + + for (String cat2 : typeStrings) { + TypeInfo commonClass = null; + boolean implicitConvertable = false; + try { + TypeInfo ti2 = TypeInfoUtils.getTypeInfoFromTypeString(cat2); + try { + commonClass = FunctionRegistry.getCommonClassForComparison(ti1, ti2); + //implicitConvertable = FunctionRegistry.implicitConvertable(ti1, ti2); + } catch (Exception err) { + System.out.println("Failed to get common class for " + ti1 + ", " + ti2 + ": " + err); + err.printStackTrace(); + //System.out.println("Unable to get TypeInfo for " + cat2 + ", skipping ..."); + } + System.out.println(cat1 + " - " + cat2 + ": " + commonClass); + //System.out.println(cat1 + " - " + cat2 + ": " + implicitConvertable); + } catch (Exception err) { + System.out.println(err); + System.out.println("Unable to get TypeInfo for " + cat2 + ", skipping ..."); + continue; + } + } + } + } + + private void unionAll(TypeInfo a, TypeInfo b, TypeInfo result) { + assertEquals(result, FunctionRegistry.getCommonClassForUnionAll(a,b)); + } + + public void testCommonClassUnionAll() { + unionAll(TypeInfoFactory.intTypeInfo, TypeInfoFactory.decimalTypeInfo, + TypeInfoFactory.decimalTypeInfo); + unionAll(TypeInfoFactory.stringTypeInfo, TypeInfoFactory.decimalTypeInfo, + TypeInfoFactory.decimalTypeInfo); + unionAll(TypeInfoFactory.doubleTypeInfo, TypeInfoFactory.decimalTypeInfo, + TypeInfoFactory.decimalTypeInfo); + unionAll(TypeInfoFactory.doubleTypeInfo, TypeInfoFactory.stringTypeInfo, + TypeInfoFactory.stringTypeInfo); + + unionAll(varchar5, varchar10, varchar10); + unionAll(varchar10, varchar5, varchar10); + unionAll(varchar10, TypeInfoFactory.stringTypeInfo, TypeInfoFactory.stringTypeInfo); + unionAll(TypeInfoFactory.stringTypeInfo, varchar10, TypeInfoFactory.stringTypeInfo); + } + + public void testGetTypeInfoForPrimitiveCategory() { + // varchar should take string length into account. + // varchar(5), varchar(10) => varchar(10) + assertEquals(varchar10, FunctionRegistry.getTypeInfoForPrimitiveCategory( + (PrimitiveTypeInfo) varchar5, (PrimitiveTypeInfo) varchar10, PrimitiveCategory.VARCHAR)); + assertEquals(varchar10, FunctionRegistry.getTypeInfoForPrimitiveCategory( + (PrimitiveTypeInfo) varchar10, (PrimitiveTypeInfo) varchar5, PrimitiveCategory.VARCHAR)); + + // non-qualified types should simply return the TypeInfo associated with that type + assertEquals(TypeInfoFactory.stringTypeInfo, FunctionRegistry.getTypeInfoForPrimitiveCategory( + (PrimitiveTypeInfo) varchar10, (PrimitiveTypeInfo) TypeInfoFactory.stringTypeInfo, + PrimitiveCategory.STRING)); } @Override diff --git ql/src/test/queries/clientpositive/alter_varchar1.q ql/src/test/queries/clientpositive/alter_varchar1.q new file mode 100644 index 0000000..fa9d99e --- /dev/null +++ ql/src/test/queries/clientpositive/alter_varchar1.q @@ -0,0 +1,32 @@ +drop table alter_varchar_1; + +create table alter_varchar_1 (key string, value string); +insert overwrite table alter_varchar_1 + select key, value from src limit 5; + +select * from alter_varchar_1; + +-- change column to varchar +alter table alter_varchar_1 change column value value varchar(20); +-- contents should still look the same +select * from alter_varchar_1; + +-- change column to smaller varchar +alter table alter_varchar_1 change column value value varchar(3); +-- value column should be truncated now +select * from alter_varchar_1; + +-- change back to bigger varchar +alter table alter_varchar_1 change column value value varchar(20); +-- column values should be full size again +select * from alter_varchar_1; + +-- add varchar column +alter table alter_varchar_1 add columns (key2 int, value2 varchar(10)); +select * from alter_varchar_1; + +insert overwrite table alter_varchar_1 + select key, value, key, value from src limit 5; +select * from alter_varchar_1; + +drop table alter_varchar_1; diff --git ql/src/test/queries/clientpositive/ctas_varchar.q ql/src/test/queries/clientpositive/ctas_varchar.q new file mode 100644 index 0000000..f646b36 --- /dev/null +++ ql/src/test/queries/clientpositive/ctas_varchar.q @@ -0,0 +1,22 @@ +drop table ctas_varchar_1; +drop table ctas_varchar_2; +drop view ctas_varchar_3; + +create table ctas_varchar_1 (key varchar(10), value string); +insert overwrite table ctas_varchar_1 + select key, value from src sort by key, value limit 5; + +-- create table as with varchar column +create table ctas_varchar_2 as select key, value from ctas_varchar_1; + +-- view with varchar column +create view ctas_varchar_3 as select key, value from ctas_varchar_2; + +select key, value from ctas_varchar_1; +select * from ctas_varchar_2; +select * from ctas_varchar_3; + + +drop table ctas_varchar_1; +drop table ctas_varchar_2; +drop view ctas_varchar_3; diff --git ql/src/test/queries/clientpositive/partition_varchar1.q ql/src/test/queries/clientpositive/partition_varchar1.q new file mode 100644 index 0000000..d700b1c --- /dev/null +++ ql/src/test/queries/clientpositive/partition_varchar1.q @@ -0,0 +1,43 @@ +drop table partition_varchar_1; + +create table partition_varchar_1 (key string, value varchar(20)) partitioned by (dt varchar(10), region int); + +insert overwrite table partition_varchar_1 partition(dt='2000-01-01', region=1) + select * from src limit 10; +insert overwrite table partition_varchar_1 partition(dt='2000-01-01', region=2) + select * from src limit 5; +insert overwrite table partition_varchar_1 partition(dt='2013-08-08', region=1) + select * from src limit 20; +insert overwrite table partition_varchar_1 partition(dt='2013-08-08', region=10) + select * from src limit 11; + +select distinct dt from partition_varchar_1; +select * from partition_varchar_1 where dt = '2000-01-01' and region = 2 order by key,value; + +-- 15 +select count(*) from partition_varchar_1 where dt = '2000-01-01'; +-- 5 +select count(*) from partition_varchar_1 where dt = '2000-01-01' and region = 2; +-- 11 +select count(*) from partition_varchar_1 where dt = '2013-08-08' and region = 10; +-- 30 +select count(*) from partition_varchar_1 where region = 1; +-- 0 +select count(*) from partition_varchar_1 where dt = '2000-01-01' and region = 3; +-- 0 +select count(*) from partition_varchar_1 where dt = '1999-01-01'; + +-- Try other comparison operations + +-- 20 +select count(*) from partition_varchar_1 where dt > '2000-01-01' and region = 1; +-- 10 +select count(*) from partition_varchar_1 where dt < '2000-01-02' and region = 1; +-- 20 +select count(*) from partition_varchar_1 where dt >= '2000-01-02' and region = 1; +-- 10 +select count(*) from partition_varchar_1 where dt <= '2000-01-01' and region = 1; +-- 20 +select count(*) from partition_varchar_1 where dt <> '2000-01-01' and region = 1; + +drop table partition_varchar_1; diff --git ql/src/test/queries/clientpositive/varchar_1.q ql/src/test/queries/clientpositive/varchar_1.q new file mode 100644 index 0000000..bda12fe --- /dev/null +++ ql/src/test/queries/clientpositive/varchar_1.q @@ -0,0 +1,32 @@ +drop table varchar1; +drop table varchar1_1; + +create table varchar1 (key varchar(10), value varchar(20)); +create table varchar1_1 (key string, value string); + +-- load from file +load data local inpath '../data/files/srcbucket0.txt' overwrite into table varchar1; +select * from varchar1 limit 2; + +-- insert overwrite, from same/different length varchar +insert overwrite table varchar1 + select cast(key as varchar(10)), cast(value as varchar(15)) from src limit 2; +select key, value from varchar1; + +-- insert overwrite, from string +insert overwrite table varchar1 + select key, value from src limit 2; +select key, value from varchar1; + +-- insert string from varchar +insert overwrite table varchar1_1 + select key, value from varchar1 limit 2; +select key, value from varchar1_1; + +-- respect string length +insert overwrite table varchar1 + select key, cast(value as varchar(3)) from src limit 2; +select key, value from varchar1; + +drop table varchar1; +drop table varchar1_1; diff --git ql/src/test/queries/clientpositive/varchar_2.q ql/src/test/queries/clientpositive/varchar_2.q new file mode 100644 index 0000000..1eb3b85 --- /dev/null +++ ql/src/test/queries/clientpositive/varchar_2.q @@ -0,0 +1,36 @@ +drop table varchar_2; + +create table varchar_2 ( + key varchar(10), + value varchar(20) +); + +insert overwrite table varchar_2 select * from src; + +select value, sum(cast(key as int)), count(*) numrows +from src +group by value +order by value asc +limit 5; + +-- should match the query from src +select value, sum(cast(key as int)), count(*) numrows +from varchar_2 +group by value +order by value asc +limit 5; + +select value, sum(cast(key as int)), count(*) numrows +from src +group by value +order by value desc +limit 5; + +-- should match the query from src +select value, sum(cast(key as int)), count(*) numrows +from varchar_2 +group by value +order by value desc +limit 5; + +drop table varchar_2; diff --git ql/src/test/queries/clientpositive/varchar_cast.q ql/src/test/queries/clientpositive/varchar_cast.q new file mode 100644 index 0000000..550f3dc --- /dev/null +++ ql/src/test/queries/clientpositive/varchar_cast.q @@ -0,0 +1,84 @@ + +-- Cast from varchar to other data types +select + cast(cast('11' as string) as tinyint), + cast(cast('11' as string) as smallint), + cast(cast('11' as string) as int), + cast(cast('11' as string) as bigint), + cast(cast('11.00' as string) as float), + cast(cast('11.00' as string) as double), + cast(cast('11.00' as string) as decimal) +from src limit 1; + +select + cast(cast('11' as varchar(10)) as tinyint), + cast(cast('11' as varchar(10)) as smallint), + cast(cast('11' as varchar(10)) as int), + cast(cast('11' as varchar(10)) as bigint), + cast(cast('11.00' as varchar(10)) as float), + cast(cast('11.00' as varchar(10)) as double), + cast(cast('11.00' as varchar(10)) as decimal) +from src limit 1; + +select + cast(cast('2011-01-01' as string) as date), + cast(cast('2011-01-01 01:02:03' as string) as timestamp) +from src limit 1; + +select + cast(cast('2011-01-01' as varchar(10)) as date), + cast(cast('2011-01-01 01:02:03' as varchar(30)) as timestamp) +from src limit 1; + +-- no tests from string/varchar to boolean, that conversion doesn't look useful +select + cast(cast('abc123' as string) as string), + cast(cast('abc123' as string) as varchar(10)) +from src limit 1; + +select + cast(cast('abc123' as varchar(10)) as string), + cast(cast('abc123' as varchar(10)) as varchar(10)) +from src limit 1; + +-- cast from other types to varchar +select + cast(cast(11 as tinyint) as string), + cast(cast(11 as smallint) as string), + cast(cast(11 as int) as string), + cast(cast(11 as bigint) as string), + cast(cast(11.00 as float) as string), + cast(cast(11.00 as double) as string), + cast(cast(11.00 as decimal) as string) +from src limit 1; + +select + cast(cast(11 as tinyint) as varchar(10)), + cast(cast(11 as smallint) as varchar(10)), + cast(cast(11 as int) as varchar(10)), + cast(cast(11 as bigint) as varchar(10)), + cast(cast(11.00 as float) as varchar(10)), + cast(cast(11.00 as double) as varchar(10)), + cast(cast(11.00 as decimal) as varchar(10)) +from src limit 1; + +select + cast(date '2011-01-01' as string), + cast(timestamp('2011-01-01 01:02:03') as string) +from src limit 1; + +select + cast(date '2011-01-01' as varchar(10)), + cast(timestamp('2011-01-01 01:02:03') as varchar(30)) +from src limit 1; + +select + cast(true as string), + cast(false as string) +from src limit 1; + +select + cast(true as varchar(10)), + cast(false as varchar(10)) +from src limit 1; + diff --git ql/src/test/queries/clientpositive/varchar_comparison.q ql/src/test/queries/clientpositive/varchar_comparison.q new file mode 100644 index 0000000..b6c6f40 --- /dev/null +++ ql/src/test/queries/clientpositive/varchar_comparison.q @@ -0,0 +1,40 @@ + +-- Should all be true +select + cast('abc' as varchar(10)) = cast('abc' as varchar(10)), + cast('abc' as varchar(10)) <= cast('abc' as varchar(10)), + cast('abc' as varchar(10)) >= cast('abc' as varchar(10)), + cast('abc' as varchar(10)) < cast('abd' as varchar(10)), + cast('abc' as varchar(10)) > cast('abb' as varchar(10)), + cast('abc' as varchar(10)) <> cast('abb' as varchar(10)) +from src limit 1; + +-- Different varchar lengths should still compare the same +select + cast('abc' as varchar(10)) = cast('abc' as varchar(3)), + cast('abc' as varchar(10)) <= cast('abc' as varchar(3)), + cast('abc' as varchar(10)) >= cast('abc' as varchar(3)), + cast('abc' as varchar(10)) < cast('abd' as varchar(3)), + cast('abc' as varchar(10)) > cast('abb' as varchar(3)), + cast('abc' as varchar(10)) <> cast('abb' as varchar(3)) +from src limit 1; + +-- Should work with string types as well +select + cast('abc' as varchar(10)) = 'abc', + cast('abc' as varchar(10)) <= 'abc', + cast('abc' as varchar(10)) >= 'abc', + cast('abc' as varchar(10)) < 'abd', + cast('abc' as varchar(10)) > 'abb', + cast('abc' as varchar(10)) <> 'abb' +from src limit 1; + +-- leading space is significant for varchar +select + cast(' abc' as varchar(10)) <> cast('abc' as varchar(10)) +from src limit 1; + +-- trailing space is significant for varchar +select + cast('abc ' as varchar(10)) <> cast('abc' as varchar(10)) +from src limit 1; diff --git ql/src/test/queries/clientpositive/varchar_join1.q ql/src/test/queries/clientpositive/varchar_join1.q new file mode 100644 index 0000000..6a19efa --- /dev/null +++ ql/src/test/queries/clientpositive/varchar_join1.q @@ -0,0 +1,35 @@ +drop table varchar_join1_vc1; +drop table varchar_join1_vc2; +drop table varchar_join1_str; + +create table varchar_join1_vc1 ( + c1 int, + c2 varchar(10) +); + +create table varchar_join1_vc2 ( + c1 int, + c2 varchar(20) +); + +create table varchar_join1_str ( + c1 int, + c2 string +); + +load data local inpath '../data/files/vc1.txt' into table varchar_join1_vc1; +load data local inpath '../data/files/vc1.txt' into table varchar_join1_vc2; +load data local inpath '../data/files/vc1.txt' into table varchar_join1_str; + +-- Join varchar with same length varchar +select * from varchar_join1_vc1 a join varchar_join1_vc1 b on (a.c2 = b.c2) order by a.c1; + +-- Join varchar with different length varchar +select * from varchar_join1_vc1 a join varchar_join1_vc2 b on (a.c2 = b.c2) order by a.c1; + +-- Join varchar with string +select * from varchar_join1_vc1 a join varchar_join1_str b on (a.c2 = b.c2) order by a.c1; + +drop table varchar_join1_vc1; +drop table varchar_join1_vc2; +drop table varchar_join1_str; diff --git ql/src/test/queries/clientpositive/varchar_nested_types.q ql/src/test/queries/clientpositive/varchar_nested_types.q new file mode 100644 index 0000000..f3f89f6 --- /dev/null +++ ql/src/test/queries/clientpositive/varchar_nested_types.q @@ -0,0 +1,53 @@ +drop table varchar_nested_1; +drop table varchar_nested_array; +drop table varchar_nested_map; +drop table varchar_nested_struct; +drop table varchar_nested_cta; +drop table varchar_nested_view; + +create table varchar_nested_1 (key int, value varchar(20)); +insert overwrite table varchar_nested_1 + select key, value from src limit 5; + +-- arrays +create table varchar_nested_array (c1 array); +insert overwrite table varchar_nested_array + select array(value, value) from varchar_nested_1; +describe varchar_nested_array; +select * from varchar_nested_array; + +-- maps +create table varchar_nested_map (c1 map); +insert overwrite table varchar_nested_map + select map(key, value) from varchar_nested_1; +describe varchar_nested_map; +select * from varchar_nested_map; + +-- structs +create table varchar_nested_struct (c1 struct); +insert overwrite table varchar_nested_struct + select named_struct('a', key, + 'b', value, + 'c', cast(value as string)) + from varchar_nested_1; +describe varchar_nested_struct; +select * from varchar_nested_struct; + +-- nested type with create table as +create table varchar_nested_cta as + select * from varchar_nested_struct; +describe varchar_nested_cta; +select * from varchar_nested_cta; + +-- nested type with view +create table varchar_nested_view as + select * from varchar_nested_struct; +describe varchar_nested_view; +select * from varchar_nested_view; + +drop table varchar_nested_1; +drop table varchar_nested_array; +drop table varchar_nested_map; +drop table varchar_nested_struct; +drop table varchar_nested_cta; +drop table varchar_nested_view; diff --git ql/src/test/queries/clientpositive/varchar_udf1.q ql/src/test/queries/clientpositive/varchar_udf1.q new file mode 100644 index 0000000..df5b7bd --- /dev/null +++ ql/src/test/queries/clientpositive/varchar_udf1.q @@ -0,0 +1,156 @@ +drop table varchar_udf_1; + +create table varchar_udf_1 (c1 string, c2 string, c3 varchar(10), c4 varchar(20)); +insert overwrite table varchar_udf_1 + select key, value, key, value from src limit 5; + +-- UDFs with varchar support +select + concat(c1, c2), + concat(c3, c4), + concat(c1, c2) = concat(c3, c4) +from varchar_udf_1 limit 1; + +select + upper(c2), + upper(c4), + upper(c2) = upper(c4) +from varchar_udf_1 limit 1; + +select + lower(c2), + lower(c4), + lower(c2) = lower(c4) +from varchar_udf_1 limit 1; + +-- Scalar UDFs +select + ascii(c2), + ascii(c4), + ascii(c2) = ascii(c4) +from varchar_udf_1 limit 1; + +select + concat_ws('|', c1, c2), + concat_ws('|', c3, c4), + concat_ws('|', c1, c2) = concat_ws('|', c3, c4) +from varchar_udf_1 limit 1; + +select + decode(encode(c2, 'US-ASCII'), 'US-ASCII'), + decode(encode(c4, 'US-ASCII'), 'US-ASCII'), + decode(encode(c2, 'US-ASCII'), 'US-ASCII') = decode(encode(c4, 'US-ASCII'), 'US-ASCII') +from varchar_udf_1 limit 1; + +select + instr(c2, '_'), + instr(c4, '_'), + instr(c2, '_') = instr(c4, '_') +from varchar_udf_1 limit 1; + +select + length(c2), + length(c4), + length(c2) = length(c4) +from varchar_udf_1 limit 1; + +select + locate('a', 'abcdabcd', 3), + locate(cast('a' as varchar(1)), cast('abcdabcd' as varchar(10)), 3), + locate('a', 'abcdabcd', 3) = locate(cast('a' as varchar(1)), cast('abcdabcd' as varchar(10)), 3) +from varchar_udf_1 limit 1; + +select + lpad(c2, 15, ' '), + lpad(c4, 15, ' '), + lpad(c2, 15, ' ') = lpad(c4, 15, ' ') +from varchar_udf_1 limit 1; + +select + ltrim(c2), + ltrim(c4), + ltrim(c2) = ltrim(c4) +from varchar_udf_1 limit 1; + +select + regexp(c2, 'val'), + regexp(c4, 'val'), + regexp(c2, 'val') = regexp(c4, 'val') +from varchar_udf_1 limit 1; + +select + regexp_extract(c2, 'val_([0-9]+)', 1), + regexp_extract(c4, 'val_([0-9]+)', 1), + regexp_extract(c2, 'val_([0-9]+)', 1) = regexp_extract(c4, 'val_([0-9]+)', 1) +from varchar_udf_1 limit 1; + +select + regexp_replace(c2, 'val', 'replaced'), + regexp_replace(c4, 'val', 'replaced'), + regexp_replace(c2, 'val', 'replaced') = regexp_replace(c4, 'val', 'replaced') +from varchar_udf_1 limit 1; + +select + reverse(c2), + reverse(c4), + reverse(c2) = reverse(c4) +from varchar_udf_1 limit 1; + +select + rpad(c2, 15, ' '), + rpad(c4, 15, ' '), + rpad(c2, 15, ' ') = rpad(c4, 15, ' ') +from varchar_udf_1 limit 1; + +select + rtrim(c2), + rtrim(c4), + rtrim(c2) = rtrim(c4) +from varchar_udf_1 limit 1; + +select + sentences('See spot run. See jane run.'), + sentences(cast('See spot run. See jane run.' as varchar(50))) +from varchar_udf_1 limit 1; + +select + split(c2, '_'), + split(c4, '_') +from varchar_udf_1 limit 1; + +select + str_to_map('a:1,b:2,c:3',',',':'), + str_to_map(cast('a:1,b:2,c:3' as varchar(20)),',',':') +from varchar_udf_1 limit 1; + +select + substr(c2, 1, 3), + substr(c4, 1, 3), + substr(c2, 1, 3) = substr(c4, 1, 3) +from varchar_udf_1 limit 1; + +select + trim(c2), + trim(c4), + trim(c2) = trim(c4) +from varchar_udf_1 limit 1; + + +-- Aggregate Functions +select + compute_stats(c2, 16), + compute_stats(c4, 16) +from varchar_udf_1; + +select + min(c2), + min(c4) +from varchar_udf_1; + +select + max(c2), + max(c4) +from varchar_udf_1; + + +drop table varchar_udf_1; diff --git ql/src/test/queries/clientpositive/varchar_union1.q ql/src/test/queries/clientpositive/varchar_union1.q new file mode 100644 index 0000000..c4ce1b5 --- /dev/null +++ ql/src/test/queries/clientpositive/varchar_union1.q @@ -0,0 +1,47 @@ +drop table varchar_union1_vc1; +drop table varchar_union1_vc2; +drop table varchar_union1_str; + +create table varchar_union1_vc1 ( + c1 int, + c2 varchar(10) +); + +create table varchar_union1_vc2 ( + c1 int, + c2 varchar(20) +); + +create table varchar_union1_str ( + c1 int, + c2 string +); + +load data local inpath '../data/files/vc1.txt' into table varchar_union1_vc1; +load data local inpath '../data/files/vc1.txt' into table varchar_union1_vc2; +load data local inpath '../data/files/vc1.txt' into table varchar_union1_str; + +-- union varchar with same length varchar +select * from ( + select * from varchar_union1_vc1 + union all + select * from varchar_union1_vc1 limit 1 +) q1; + +-- union varchar with different length varchar +select * from ( + select * from varchar_union1_vc1 + union all + select * from varchar_union1_vc2 limit 1 +) q1; + +-- union varchar with string +select * from ( + select * from varchar_union1_vc1 + union all + select * from varchar_union1_str limit 1 +) q1; + +drop table varchar_union1_vc1; +drop table varchar_union1_vc2; +drop table varchar_union1_str; diff --git ql/src/test/results/clientpositive/alter_varchar1.q.out ql/src/test/results/clientpositive/alter_varchar1.q.out new file mode 100644 index 0000000..20cdf99 --- /dev/null +++ ql/src/test/results/clientpositive/alter_varchar1.q.out @@ -0,0 +1,199 @@ +PREHOOK: query: drop table alter_varchar_1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table alter_varchar_1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table alter_varchar_1 (key string, value string) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table alter_varchar_1 (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@alter_varchar_1 +PREHOOK: query: insert overwrite table alter_varchar_1 + select key, value from src limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@alter_varchar_1 +POSTHOOK: query: insert overwrite table alter_varchar_1 + select key, value from src limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@alter_varchar_1 +POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from alter_varchar_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_varchar_1 +#### A masked pattern was here #### +POSTHOOK: query: select * from alter_varchar_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_varchar_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +238 val_238 +86 val_86 +311 val_311 +27 val_27 +165 val_165 +PREHOOK: query: -- change column to varchar +alter table alter_varchar_1 change column value value varchar(20) +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@alter_varchar_1 +PREHOOK: Output: default@alter_varchar_1 +POSTHOOK: query: -- change column to varchar +alter table alter_varchar_1 change column value value varchar(20) +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@alter_varchar_1 +POSTHOOK: Output: default@alter_varchar_1 +POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- contents should still look the same +select * from alter_varchar_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_varchar_1 +#### A masked pattern was here #### +POSTHOOK: query: -- contents should still look the same +select * from alter_varchar_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_varchar_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +238 val_238 +86 val_86 +311 val_311 +27 val_27 +165 val_165 +PREHOOK: query: -- change column to smaller varchar +alter table alter_varchar_1 change column value value varchar(3) +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@alter_varchar_1 +PREHOOK: Output: default@alter_varchar_1 +POSTHOOK: query: -- change column to smaller varchar +alter table alter_varchar_1 change column value value varchar(3) +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@alter_varchar_1 +POSTHOOK: Output: default@alter_varchar_1 +POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- value column should be truncated now +select * from alter_varchar_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_varchar_1 +#### A masked pattern was here #### +POSTHOOK: query: -- value column should be truncated now +select * from alter_varchar_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_varchar_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +238 val +86 val +311 val +27 val +165 val +PREHOOK: query: -- change back to bigger varchar +alter table alter_varchar_1 change column value value varchar(20) +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@alter_varchar_1 +PREHOOK: Output: default@alter_varchar_1 +POSTHOOK: query: -- change back to bigger varchar +alter table alter_varchar_1 change column value value varchar(20) +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@alter_varchar_1 +POSTHOOK: Output: default@alter_varchar_1 +POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- column values should be full size again +select * from alter_varchar_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_varchar_1 +#### A masked pattern was here #### +POSTHOOK: query: -- column values should be full size again +select * from alter_varchar_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_varchar_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +238 val_238 +86 val_86 +311 val_311 +27 val_27 +165 val_165 +PREHOOK: query: -- add varchar column +alter table alter_varchar_1 add columns (key2 int, value2 varchar(10)) +PREHOOK: type: ALTERTABLE_ADDCOLS +PREHOOK: Input: default@alter_varchar_1 +PREHOOK: Output: default@alter_varchar_1 +POSTHOOK: query: -- add varchar column +alter table alter_varchar_1 add columns (key2 int, value2 varchar(10)) +POSTHOOK: type: ALTERTABLE_ADDCOLS +POSTHOOK: Input: default@alter_varchar_1 +POSTHOOK: Output: default@alter_varchar_1 +POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from alter_varchar_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_varchar_1 +#### A masked pattern was here #### +POSTHOOK: query: select * from alter_varchar_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_varchar_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +238 val_238 NULL NULL +86 val_86 NULL NULL +311 val_311 NULL NULL +27 val_27 NULL NULL +165 val_165 NULL NULL +PREHOOK: query: insert overwrite table alter_varchar_1 + select key, value, key, value from src limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@alter_varchar_1 +POSTHOOK: query: insert overwrite table alter_varchar_1 + select key, value, key, value from src limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@alter_varchar_1 +POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_varchar_1.key2 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_varchar_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_varchar_1.value2 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from alter_varchar_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_varchar_1 +#### A masked pattern was here #### +POSTHOOK: query: select * from alter_varchar_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_varchar_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_varchar_1.key2 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_varchar_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_varchar_1.value2 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +238 val_238 238 val_238 +86 val_86 86 val_86 +311 val_311 311 val_311 +27 val_27 27 val_27 +165 val_165 165 val_165 +PREHOOK: query: drop table alter_varchar_1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@alter_varchar_1 +PREHOOK: Output: default@alter_varchar_1 +POSTHOOK: query: drop table alter_varchar_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@alter_varchar_1 +POSTHOOK: Output: default@alter_varchar_1 +POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_varchar_1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_varchar_1.key2 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_varchar_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_varchar_1.value2 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] diff --git ql/src/test/results/clientpositive/ctas_varchar.q.out ql/src/test/results/clientpositive/ctas_varchar.q.out new file mode 100644 index 0000000..a4a9c66 --- /dev/null +++ ql/src/test/results/clientpositive/ctas_varchar.q.out @@ -0,0 +1,126 @@ +PREHOOK: query: drop table ctas_varchar_1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table ctas_varchar_1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table ctas_varchar_2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table ctas_varchar_2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop view ctas_varchar_3 +PREHOOK: type: DROPVIEW +POSTHOOK: query: drop view ctas_varchar_3 +POSTHOOK: type: DROPVIEW +PREHOOK: query: create table ctas_varchar_1 (key varchar(10), value string) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table ctas_varchar_1 (key varchar(10), value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@ctas_varchar_1 +PREHOOK: query: insert overwrite table ctas_varchar_1 + select key, value from src sort by key, value limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@ctas_varchar_1 +POSTHOOK: query: insert overwrite table ctas_varchar_1 + select key, value from src sort by key, value limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@ctas_varchar_1 +POSTHOOK: Lineage: ctas_varchar_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- create table as with varchar column +create table ctas_varchar_2 as select key, value from ctas_varchar_1 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@ctas_varchar_1 +POSTHOOK: query: -- create table as with varchar column +create table ctas_varchar_2 as select key, value from ctas_varchar_1 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@ctas_varchar_1 +POSTHOOK: Output: default@ctas_varchar_2 +POSTHOOK: Lineage: ctas_varchar_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- view with varchar column +create view ctas_varchar_3 as select key, value from ctas_varchar_2 +PREHOOK: type: CREATEVIEW +POSTHOOK: query: -- view with varchar column +create view ctas_varchar_3 as select key, value from ctas_varchar_2 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Output: default@ctas_varchar_3 +POSTHOOK: Lineage: ctas_varchar_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select key, value from ctas_varchar_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@ctas_varchar_1 +#### A masked pattern was here #### +POSTHOOK: query: select key, value from ctas_varchar_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ctas_varchar_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: ctas_varchar_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +0 val_0 +0 val_0 +0 val_0 +10 val_10 +100 val_100 +PREHOOK: query: select * from ctas_varchar_2 +PREHOOK: type: QUERY +PREHOOK: Input: default@ctas_varchar_2 +#### A masked pattern was here #### +POSTHOOK: query: select * from ctas_varchar_2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ctas_varchar_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: ctas_varchar_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +0 val_0 +0 val_0 +0 val_0 +10 val_10 +100 val_100 +PREHOOK: query: select * from ctas_varchar_3 +PREHOOK: type: QUERY +PREHOOK: Input: default@ctas_varchar_2 +PREHOOK: Input: default@ctas_varchar_3 +#### A masked pattern was here #### +POSTHOOK: query: select * from ctas_varchar_3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ctas_varchar_2 +POSTHOOK: Input: default@ctas_varchar_3 +#### A masked pattern was here #### +POSTHOOK: Lineage: ctas_varchar_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +0 val_0 +0 val_0 +0 val_0 +10 val_10 +100 val_100 +PREHOOK: query: drop table ctas_varchar_1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ctas_varchar_1 +PREHOOK: Output: default@ctas_varchar_1 +POSTHOOK: query: drop table ctas_varchar_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ctas_varchar_1 +POSTHOOK: Output: default@ctas_varchar_1 +POSTHOOK: Lineage: ctas_varchar_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: drop table ctas_varchar_2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ctas_varchar_2 +PREHOOK: Output: default@ctas_varchar_2 +POSTHOOK: query: drop table ctas_varchar_2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ctas_varchar_2 +POSTHOOK: Output: default@ctas_varchar_2 +POSTHOOK: Lineage: ctas_varchar_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: drop view ctas_varchar_3 +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@ctas_varchar_3 +PREHOOK: Output: default@ctas_varchar_3 +POSTHOOK: query: drop view ctas_varchar_3 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@ctas_varchar_3 +POSTHOOK: Output: default@ctas_varchar_3 +POSTHOOK: Lineage: ctas_varchar_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_varchar_1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] diff --git ql/src/test/results/clientpositive/partition_varchar1.q.out ql/src/test/results/clientpositive/partition_varchar1.q.out new file mode 100644 index 0000000..e7fef01 --- /dev/null +++ ql/src/test/results/clientpositive/partition_varchar1.q.out @@ -0,0 +1,369 @@ +PREHOOK: query: drop table partition_varchar_1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table partition_varchar_1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table partition_varchar_1 (key string, value varchar(20)) partitioned by (dt varchar(10), region int) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table partition_varchar_1 (key string, value varchar(20)) partitioned by (dt varchar(10), region int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@partition_varchar_1 +PREHOOK: query: insert overwrite table partition_varchar_1 partition(dt='2000-01-01', region=1) + select * from src limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@partition_varchar_1@dt=2000-01-01/region=1 +POSTHOOK: query: insert overwrite table partition_varchar_1 partition(dt='2000-01-01', region=1) + select * from src limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@partition_varchar_1@dt=2000-01-01/region=1 +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table partition_varchar_1 partition(dt='2000-01-01', region=2) + select * from src limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@partition_varchar_1@dt=2000-01-01/region=2 +POSTHOOK: query: insert overwrite table partition_varchar_1 partition(dt='2000-01-01', region=2) + select * from src limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@partition_varchar_1@dt=2000-01-01/region=2 +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table partition_varchar_1 partition(dt='2013-08-08', region=1) + select * from src limit 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@partition_varchar_1@dt=2013-08-08/region=1 +POSTHOOK: query: insert overwrite table partition_varchar_1 partition(dt='2013-08-08', region=1) + select * from src limit 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@partition_varchar_1@dt=2013-08-08/region=1 +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table partition_varchar_1 partition(dt='2013-08-08', region=10) + select * from src limit 11 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@partition_varchar_1@dt=2013-08-08/region=10 +POSTHOOK: query: insert overwrite table partition_varchar_1 partition(dt='2013-08-08', region=10) + select * from src limit 11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@partition_varchar_1@dt=2013-08-08/region=10 +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select distinct dt from partition_varchar_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_varchar_1 +PREHOOK: Input: default@partition_varchar_1@dt=2000-01-01/region=1 +PREHOOK: Input: default@partition_varchar_1@dt=2000-01-01/region=2 +PREHOOK: Input: default@partition_varchar_1@dt=2013-08-08/region=1 +PREHOOK: Input: default@partition_varchar_1@dt=2013-08-08/region=10 +#### A masked pattern was here #### +POSTHOOK: query: select distinct dt from partition_varchar_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_varchar_1 +POSTHOOK: Input: default@partition_varchar_1@dt=2000-01-01/region=1 +POSTHOOK: Input: default@partition_varchar_1@dt=2000-01-01/region=2 +POSTHOOK: Input: default@partition_varchar_1@dt=2013-08-08/region=1 +POSTHOOK: Input: default@partition_varchar_1@dt=2013-08-08/region=10 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +2000-01-01 +2013-08-08 +PREHOOK: query: select * from partition_varchar_1 where dt = '2000-01-01' and region = 2 order by key,value +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_varchar_1 +PREHOOK: Input: default@partition_varchar_1@dt=2000-01-01/region=2 +#### A masked pattern was here #### +POSTHOOK: query: select * from partition_varchar_1 where dt = '2000-01-01' and region = 2 order by key,value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_varchar_1 +POSTHOOK: Input: default@partition_varchar_1@dt=2000-01-01/region=2 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +165 val_165 2000-01-01 2 +238 val_238 2000-01-01 2 +27 val_27 2000-01-01 2 +311 val_311 2000-01-01 2 +86 val_86 2000-01-01 2 +PREHOOK: query: -- 15 +select count(*) from partition_varchar_1 where dt = '2000-01-01' +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_varchar_1 +PREHOOK: Input: default@partition_varchar_1@dt=2000-01-01/region=1 +PREHOOK: Input: default@partition_varchar_1@dt=2000-01-01/region=2 +#### A masked pattern was here #### +POSTHOOK: query: -- 15 +select count(*) from partition_varchar_1 where dt = '2000-01-01' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_varchar_1 +POSTHOOK: Input: default@partition_varchar_1@dt=2000-01-01/region=1 +POSTHOOK: Input: default@partition_varchar_1@dt=2000-01-01/region=2 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +15 +PREHOOK: query: -- 5 +select count(*) from partition_varchar_1 where dt = '2000-01-01' and region = 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_varchar_1 +PREHOOK: Input: default@partition_varchar_1@dt=2000-01-01/region=2 +#### A masked pattern was here #### +POSTHOOK: query: -- 5 +select count(*) from partition_varchar_1 where dt = '2000-01-01' and region = 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_varchar_1 +POSTHOOK: Input: default@partition_varchar_1@dt=2000-01-01/region=2 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +5 +PREHOOK: query: -- 11 +select count(*) from partition_varchar_1 where dt = '2013-08-08' and region = 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_varchar_1 +PREHOOK: Input: default@partition_varchar_1@dt=2013-08-08/region=10 +#### A masked pattern was here #### +POSTHOOK: query: -- 11 +select count(*) from partition_varchar_1 where dt = '2013-08-08' and region = 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_varchar_1 +POSTHOOK: Input: default@partition_varchar_1@dt=2013-08-08/region=10 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +11 +PREHOOK: query: -- 30 +select count(*) from partition_varchar_1 where region = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_varchar_1 +PREHOOK: Input: default@partition_varchar_1@dt=2000-01-01/region=1 +PREHOOK: Input: default@partition_varchar_1@dt=2013-08-08/region=1 +#### A masked pattern was here #### +POSTHOOK: query: -- 30 +select count(*) from partition_varchar_1 where region = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_varchar_1 +POSTHOOK: Input: default@partition_varchar_1@dt=2000-01-01/region=1 +POSTHOOK: Input: default@partition_varchar_1@dt=2013-08-08/region=1 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +30 +PREHOOK: query: -- 0 +select count(*) from partition_varchar_1 where dt = '2000-01-01' and region = 3 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_varchar_1 +#### A masked pattern was here #### +POSTHOOK: query: -- 0 +select count(*) from partition_varchar_1 where dt = '2000-01-01' and region = 3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_varchar_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +0 +PREHOOK: query: -- 0 +select count(*) from partition_varchar_1 where dt = '1999-01-01' +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_varchar_1 +#### A masked pattern was here #### +POSTHOOK: query: -- 0 +select count(*) from partition_varchar_1 where dt = '1999-01-01' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_varchar_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +0 +PREHOOK: query: -- Try other comparison operations + +-- 20 +select count(*) from partition_varchar_1 where dt > '2000-01-01' and region = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_varchar_1 +PREHOOK: Input: default@partition_varchar_1@dt=2013-08-08/region=1 +#### A masked pattern was here #### +POSTHOOK: query: -- Try other comparison operations + +-- 20 +select count(*) from partition_varchar_1 where dt > '2000-01-01' and region = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_varchar_1 +POSTHOOK: Input: default@partition_varchar_1@dt=2013-08-08/region=1 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +20 +PREHOOK: query: -- 10 +select count(*) from partition_varchar_1 where dt < '2000-01-02' and region = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_varchar_1 +PREHOOK: Input: default@partition_varchar_1@dt=2000-01-01/region=1 +#### A masked pattern was here #### +POSTHOOK: query: -- 10 +select count(*) from partition_varchar_1 where dt < '2000-01-02' and region = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_varchar_1 +POSTHOOK: Input: default@partition_varchar_1@dt=2000-01-01/region=1 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +10 +PREHOOK: query: -- 20 +select count(*) from partition_varchar_1 where dt >= '2000-01-02' and region = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_varchar_1 +PREHOOK: Input: default@partition_varchar_1@dt=2013-08-08/region=1 +#### A masked pattern was here #### +POSTHOOK: query: -- 20 +select count(*) from partition_varchar_1 where dt >= '2000-01-02' and region = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_varchar_1 +POSTHOOK: Input: default@partition_varchar_1@dt=2013-08-08/region=1 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +20 +PREHOOK: query: -- 10 +select count(*) from partition_varchar_1 where dt <= '2000-01-01' and region = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_varchar_1 +PREHOOK: Input: default@partition_varchar_1@dt=2000-01-01/region=1 +#### A masked pattern was here #### +POSTHOOK: query: -- 10 +select count(*) from partition_varchar_1 where dt <= '2000-01-01' and region = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_varchar_1 +POSTHOOK: Input: default@partition_varchar_1@dt=2000-01-01/region=1 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +10 +PREHOOK: query: -- 20 +select count(*) from partition_varchar_1 where dt <> '2000-01-01' and region = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_varchar_1 +PREHOOK: Input: default@partition_varchar_1@dt=2013-08-08/region=1 +#### A masked pattern was here #### +POSTHOOK: query: -- 20 +select count(*) from partition_varchar_1 where dt <> '2000-01-01' and region = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_varchar_1 +POSTHOOK: Input: default@partition_varchar_1@dt=2013-08-08/region=1 +#### A masked pattern was here #### +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +20 +PREHOOK: query: drop table partition_varchar_1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@partition_varchar_1 +PREHOOK: Output: default@partition_varchar_1 +POSTHOOK: query: drop table partition_varchar_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@partition_varchar_1 +POSTHOOK: Output: default@partition_varchar_1 +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2000-01-01,region=2).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=10).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_varchar_1 PARTITION(dt=2013-08-08,region=1).value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] diff --git ql/src/test/results/clientpositive/varchar_1.q.out ql/src/test/results/clientpositive/varchar_1.q.out new file mode 100644 index 0000000..18d0b8f --- /dev/null +++ ql/src/test/results/clientpositive/varchar_1.q.out @@ -0,0 +1,196 @@ +PREHOOK: query: drop table varchar1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table varchar1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table varchar1_1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table varchar1_1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table varchar1 (key varchar(10), value varchar(20)) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table varchar1 (key varchar(10), value varchar(20)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@varchar1 +PREHOOK: query: create table varchar1_1 (key string, value string) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table varchar1_1 (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@varchar1_1 +PREHOOK: query: -- load from file +load data local inpath '../data/files/srcbucket0.txt' overwrite into table varchar1 +PREHOOK: type: LOAD +PREHOOK: Output: default@varchar1 +POSTHOOK: query: -- load from file +load data local inpath '../data/files/srcbucket0.txt' overwrite into table varchar1 +POSTHOOK: type: LOAD +POSTHOOK: Output: default@varchar1 +PREHOOK: query: select * from varchar1 limit 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar1 +#### A masked pattern was here #### +POSTHOOK: query: select * from varchar1 limit 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar1 +#### A masked pattern was here #### +474 val_475 +62 val_63 +PREHOOK: query: -- insert overwrite, from same/different length varchar +insert overwrite table varchar1 + select cast(key as varchar(10)), cast(value as varchar(15)) from src limit 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@varchar1 +POSTHOOK: query: -- insert overwrite, from same/different length varchar +insert overwrite table varchar1 + select cast(key as varchar(10)), cast(value as varchar(15)) from src limit 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@varchar1 +POSTHOOK: Lineage: varchar1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select key, value from varchar1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar1 +#### A masked pattern was here #### +POSTHOOK: query: select key, value from varchar1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +238 val_238 +86 val_86 +PREHOOK: query: -- insert overwrite, from string +insert overwrite table varchar1 + select key, value from src limit 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@varchar1 +POSTHOOK: query: -- insert overwrite, from string +insert overwrite table varchar1 + select key, value from src limit 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@varchar1 +POSTHOOK: Lineage: varchar1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select key, value from varchar1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar1 +#### A masked pattern was here #### +POSTHOOK: query: select key, value from varchar1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +238 val_238 +86 val_86 +PREHOOK: query: -- insert string from varchar +insert overwrite table varchar1_1 + select key, value from varchar1 limit 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar1 +PREHOOK: Output: default@varchar1_1 +POSTHOOK: query: -- insert string from varchar +insert overwrite table varchar1_1 + select key, value from varchar1 limit 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar1 +POSTHOOK: Output: default@varchar1_1 +POSTHOOK: Lineage: varchar1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1_1.key SIMPLE [(varchar1)varchar1.FieldSchema(name:key, type:varchar(10), comment:null), ] +POSTHOOK: Lineage: varchar1_1.value SIMPLE [(varchar1)varchar1.FieldSchema(name:value, type:varchar(20), comment:null), ] +PREHOOK: query: select key, value from varchar1_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar1_1 +#### A masked pattern was here #### +POSTHOOK: query: select key, value from varchar1_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar1_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1_1.key SIMPLE [(varchar1)varchar1.FieldSchema(name:key, type:varchar(10), comment:null), ] +POSTHOOK: Lineage: varchar1_1.value SIMPLE [(varchar1)varchar1.FieldSchema(name:value, type:varchar(20), comment:null), ] +238 val_238 +86 val_86 +PREHOOK: query: -- respect string length +insert overwrite table varchar1 + select key, cast(value as varchar(3)) from src limit 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@varchar1 +POSTHOOK: query: -- respect string length +insert overwrite table varchar1 + select key, cast(value as varchar(3)) from src limit 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@varchar1 +POSTHOOK: Lineage: varchar1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1_1.key SIMPLE [(varchar1)varchar1.FieldSchema(name:key, type:varchar(10), comment:null), ] +POSTHOOK: Lineage: varchar1_1.value SIMPLE [(varchar1)varchar1.FieldSchema(name:value, type:varchar(20), comment:null), ] +PREHOOK: query: select key, value from varchar1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar1 +#### A masked pattern was here #### +POSTHOOK: query: select key, value from varchar1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1_1.key SIMPLE [(varchar1)varchar1.FieldSchema(name:key, type:varchar(10), comment:null), ] +POSTHOOK: Lineage: varchar1_1.value SIMPLE [(varchar1)varchar1.FieldSchema(name:value, type:varchar(20), comment:null), ] +238 val +86 val +PREHOOK: query: drop table varchar1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@varchar1 +PREHOOK: Output: default@varchar1 +POSTHOOK: query: drop table varchar1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@varchar1 +POSTHOOK: Output: default@varchar1 +POSTHOOK: Lineage: varchar1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1_1.key SIMPLE [(varchar1)varchar1.FieldSchema(name:key, type:varchar(10), comment:null), ] +POSTHOOK: Lineage: varchar1_1.value SIMPLE [(varchar1)varchar1.FieldSchema(name:value, type:varchar(20), comment:null), ] +PREHOOK: query: drop table varchar1_1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@varchar1_1 +PREHOOK: Output: default@varchar1_1 +POSTHOOK: query: drop table varchar1_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@varchar1_1 +POSTHOOK: Output: default@varchar1_1 +POSTHOOK: Lineage: varchar1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar1_1.key SIMPLE [(varchar1)varchar1.FieldSchema(name:key, type:varchar(10), comment:null), ] +POSTHOOK: Lineage: varchar1_1.value SIMPLE [(varchar1)varchar1.FieldSchema(name:value, type:varchar(20), comment:null), ] diff --git ql/src/test/results/clientpositive/varchar_2.q.out ql/src/test/results/clientpositive/varchar_2.q.out new file mode 100644 index 0000000..50ab30f --- /dev/null +++ ql/src/test/results/clientpositive/varchar_2.q.out @@ -0,0 +1,131 @@ +PREHOOK: query: drop table varchar_2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table varchar_2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table varchar_2 ( + key varchar(10), + value varchar(20) +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table varchar_2 ( + key varchar(10), + value varchar(20) +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@varchar_2 +PREHOOK: query: insert overwrite table varchar_2 select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@varchar_2 +POSTHOOK: query: insert overwrite table varchar_2 select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@varchar_2 +POSTHOOK: Lineage: varchar_2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_2.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select value, sum(cast(key as int)), count(*) numrows +from src +group by value +order by value asc +limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select value, sum(cast(key as int)), count(*) numrows +from src +group by value +order by value asc +limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_2.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +val_0 0 3 +val_10 10 1 +val_100 200 2 +val_103 206 2 +val_104 208 2 +PREHOOK: query: -- should match the query from src +select value, sum(cast(key as int)), count(*) numrows +from varchar_2 +group by value +order by value asc +limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_2 +#### A masked pattern was here #### +POSTHOOK: query: -- should match the query from src +select value, sum(cast(key as int)), count(*) numrows +from varchar_2 +group by value +order by value asc +limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_2.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +val_0 0 3 +val_10 10 1 +val_100 200 2 +val_103 206 2 +val_104 208 2 +PREHOOK: query: select value, sum(cast(key as int)), count(*) numrows +from src +group by value +order by value desc +limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select value, sum(cast(key as int)), count(*) numrows +from src +group by value +order by value desc +limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_2.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +val_98 196 2 +val_97 194 2 +val_96 96 1 +val_95 190 2 +val_92 92 1 +PREHOOK: query: -- should match the query from src +select value, sum(cast(key as int)), count(*) numrows +from varchar_2 +group by value +order by value desc +limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_2 +#### A masked pattern was here #### +POSTHOOK: query: -- should match the query from src +select value, sum(cast(key as int)), count(*) numrows +from varchar_2 +group by value +order by value desc +limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_2.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +val_98 196 2 +val_97 194 2 +val_96 96 1 +val_95 190 2 +val_92 92 1 +PREHOOK: query: drop table varchar_2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@varchar_2 +PREHOOK: Output: default@varchar_2 +POSTHOOK: query: drop table varchar_2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@varchar_2 +POSTHOOK: Output: default@varchar_2 +POSTHOOK: Lineage: varchar_2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_2.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] diff --git ql/src/test/results/clientpositive/varchar_cast.q.out ql/src/test/results/clientpositive/varchar_cast.q.out new file mode 100644 index 0000000..5a968f2 --- /dev/null +++ ql/src/test/results/clientpositive/varchar_cast.q.out @@ -0,0 +1,226 @@ +PREHOOK: query: -- Cast from varchar to other data types +select + cast(cast('11' as string) as tinyint), + cast(cast('11' as string) as smallint), + cast(cast('11' as string) as int), + cast(cast('11' as string) as bigint), + cast(cast('11.00' as string) as float), + cast(cast('11.00' as string) as double), + cast(cast('11.00' as string) as decimal) +from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: -- Cast from varchar to other data types +select + cast(cast('11' as string) as tinyint), + cast(cast('11' as string) as smallint), + cast(cast('11' as string) as int), + cast(cast('11' as string) as bigint), + cast(cast('11.00' as string) as float), + cast(cast('11.00' as string) as double), + cast(cast('11.00' as string) as decimal) +from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +11 11 11 11 11.0 11.0 11 +PREHOOK: query: select + cast(cast('11' as varchar(10)) as tinyint), + cast(cast('11' as varchar(10)) as smallint), + cast(cast('11' as varchar(10)) as int), + cast(cast('11' as varchar(10)) as bigint), + cast(cast('11.00' as varchar(10)) as float), + cast(cast('11.00' as varchar(10)) as double), + cast(cast('11.00' as varchar(10)) as decimal) +from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select + cast(cast('11' as varchar(10)) as tinyint), + cast(cast('11' as varchar(10)) as smallint), + cast(cast('11' as varchar(10)) as int), + cast(cast('11' as varchar(10)) as bigint), + cast(cast('11.00' as varchar(10)) as float), + cast(cast('11.00' as varchar(10)) as double), + cast(cast('11.00' as varchar(10)) as decimal) +from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +11 11 11 11 11.0 11.0 11 +PREHOOK: query: select + cast(cast('2011-01-01' as string) as date), + cast(cast('2011-01-01 01:02:03' as string) as timestamp) +from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select + cast(cast('2011-01-01' as string) as date), + cast(cast('2011-01-01 01:02:03' as string) as timestamp) +from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +2011-01-01 2011-01-01 01:02:03 +PREHOOK: query: select + cast(cast('2011-01-01' as varchar(10)) as date), + cast(cast('2011-01-01 01:02:03' as varchar(30)) as timestamp) +from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select + cast(cast('2011-01-01' as varchar(10)) as date), + cast(cast('2011-01-01 01:02:03' as varchar(30)) as timestamp) +from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +2011-01-01 2011-01-01 01:02:03 +PREHOOK: query: -- no tests from string/varchar to boolean, that conversion doesn't look useful +select + cast(cast('abc123' as string) as string), + cast(cast('abc123' as string) as varchar(10)) +from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: -- no tests from string/varchar to boolean, that conversion doesn't look useful +select + cast(cast('abc123' as string) as string), + cast(cast('abc123' as string) as varchar(10)) +from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +abc123 abc123 +PREHOOK: query: select + cast(cast('abc123' as varchar(10)) as string), + cast(cast('abc123' as varchar(10)) as varchar(10)) +from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select + cast(cast('abc123' as varchar(10)) as string), + cast(cast('abc123' as varchar(10)) as varchar(10)) +from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +abc123 abc123 +PREHOOK: query: -- cast from other types to varchar +select + cast(cast(11 as tinyint) as string), + cast(cast(11 as smallint) as string), + cast(cast(11 as int) as string), + cast(cast(11 as bigint) as string), + cast(cast(11.00 as float) as string), + cast(cast(11.00 as double) as string), + cast(cast(11.00 as decimal) as string) +from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: -- cast from other types to varchar +select + cast(cast(11 as tinyint) as string), + cast(cast(11 as smallint) as string), + cast(cast(11 as int) as string), + cast(cast(11 as bigint) as string), + cast(cast(11.00 as float) as string), + cast(cast(11.00 as double) as string), + cast(cast(11.00 as decimal) as string) +from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +11 11 11 11 11.0 11.0 11 +PREHOOK: query: select + cast(cast(11 as tinyint) as varchar(10)), + cast(cast(11 as smallint) as varchar(10)), + cast(cast(11 as int) as varchar(10)), + cast(cast(11 as bigint) as varchar(10)), + cast(cast(11.00 as float) as varchar(10)), + cast(cast(11.00 as double) as varchar(10)), + cast(cast(11.00 as decimal) as varchar(10)) +from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select + cast(cast(11 as tinyint) as varchar(10)), + cast(cast(11 as smallint) as varchar(10)), + cast(cast(11 as int) as varchar(10)), + cast(cast(11 as bigint) as varchar(10)), + cast(cast(11.00 as float) as varchar(10)), + cast(cast(11.00 as double) as varchar(10)), + cast(cast(11.00 as decimal) as varchar(10)) +from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +11 11 11 11 11.0 11.0 11 +PREHOOK: query: select + cast(date '2011-01-01' as string), + cast(timestamp('2011-01-01 01:02:03') as string) +from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select + cast(date '2011-01-01' as string), + cast(timestamp('2011-01-01 01:02:03') as string) +from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +2011-01-01 2011-01-01 01:02:03 +PREHOOK: query: select + cast(date '2011-01-01' as varchar(10)), + cast(timestamp('2011-01-01 01:02:03') as varchar(30)) +from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select + cast(date '2011-01-01' as varchar(10)), + cast(timestamp('2011-01-01 01:02:03') as varchar(30)) +from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +2011-01-01 2011-01-01 01:02:03 +PREHOOK: query: select + cast(true as string), + cast(false as string) +from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select + cast(true as string), + cast(false as string) +from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +TRUE FALSE +PREHOOK: query: select + cast(true as varchar(10)), + cast(false as varchar(10)) +from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select + cast(true as varchar(10)), + cast(false as varchar(10)) +from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +TRUE FALSE diff --git ql/src/test/results/clientpositive/varchar_comparison.q.out ql/src/test/results/clientpositive/varchar_comparison.q.out new file mode 100644 index 0000000..e2c7aaf --- /dev/null +++ ql/src/test/results/clientpositive/varchar_comparison.q.out @@ -0,0 +1,105 @@ +PREHOOK: query: -- Should all be true +select + cast('abc' as varchar(10)) = cast('abc' as varchar(10)), + cast('abc' as varchar(10)) <= cast('abc' as varchar(10)), + cast('abc' as varchar(10)) >= cast('abc' as varchar(10)), + cast('abc' as varchar(10)) < cast('abd' as varchar(10)), + cast('abc' as varchar(10)) > cast('abb' as varchar(10)), + cast('abc' as varchar(10)) <> cast('abb' as varchar(10)) +from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: -- Should all be true +select + cast('abc' as varchar(10)) = cast('abc' as varchar(10)), + cast('abc' as varchar(10)) <= cast('abc' as varchar(10)), + cast('abc' as varchar(10)) >= cast('abc' as varchar(10)), + cast('abc' as varchar(10)) < cast('abd' as varchar(10)), + cast('abc' as varchar(10)) > cast('abb' as varchar(10)), + cast('abc' as varchar(10)) <> cast('abb' as varchar(10)) +from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +true true true true true true +PREHOOK: query: -- Different varchar lengths should still compare the same +select + cast('abc' as varchar(10)) = cast('abc' as varchar(3)), + cast('abc' as varchar(10)) <= cast('abc' as varchar(3)), + cast('abc' as varchar(10)) >= cast('abc' as varchar(3)), + cast('abc' as varchar(10)) < cast('abd' as varchar(3)), + cast('abc' as varchar(10)) > cast('abb' as varchar(3)), + cast('abc' as varchar(10)) <> cast('abb' as varchar(3)) +from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: -- Different varchar lengths should still compare the same +select + cast('abc' as varchar(10)) = cast('abc' as varchar(3)), + cast('abc' as varchar(10)) <= cast('abc' as varchar(3)), + cast('abc' as varchar(10)) >= cast('abc' as varchar(3)), + cast('abc' as varchar(10)) < cast('abd' as varchar(3)), + cast('abc' as varchar(10)) > cast('abb' as varchar(3)), + cast('abc' as varchar(10)) <> cast('abb' as varchar(3)) +from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +true true true true true true +PREHOOK: query: -- Should work with string types as well +select + cast('abc' as varchar(10)) = 'abc', + cast('abc' as varchar(10)) <= 'abc', + cast('abc' as varchar(10)) >= 'abc', + cast('abc' as varchar(10)) < 'abd', + cast('abc' as varchar(10)) > 'abb', + cast('abc' as varchar(10)) <> 'abb' +from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: -- Should work with string types as well +select + cast('abc' as varchar(10)) = 'abc', + cast('abc' as varchar(10)) <= 'abc', + cast('abc' as varchar(10)) >= 'abc', + cast('abc' as varchar(10)) < 'abd', + cast('abc' as varchar(10)) > 'abb', + cast('abc' as varchar(10)) <> 'abb' +from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +true true true true true true +PREHOOK: query: -- leading space is significant for varchar +select + cast(' abc' as varchar(10)) <> cast('abc' as varchar(10)) +from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: -- leading space is significant for varchar +select + cast(' abc' as varchar(10)) <> cast('abc' as varchar(10)) +from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +true +PREHOOK: query: -- trailing space is significant for varchar +select + cast('abc ' as varchar(10)) <> cast('abc' as varchar(10)) +from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: -- trailing space is significant for varchar +select + cast('abc ' as varchar(10)) <> cast('abc' as varchar(10)) +from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +true diff --git ql/src/test/results/clientpositive/varchar_join1.q.out ql/src/test/results/clientpositive/varchar_join1.q.out new file mode 100644 index 0000000..b389de9 --- /dev/null +++ ql/src/test/results/clientpositive/varchar_join1.q.out @@ -0,0 +1,130 @@ +PREHOOK: query: drop table varchar_join1_vc1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table varchar_join1_vc1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table varchar_join1_vc2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table varchar_join1_vc2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table varchar_join1_str +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table varchar_join1_str +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table varchar_join1_vc1 ( + c1 int, + c2 varchar(10) +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table varchar_join1_vc1 ( + c1 int, + c2 varchar(10) +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@varchar_join1_vc1 +PREHOOK: query: create table varchar_join1_vc2 ( + c1 int, + c2 varchar(20) +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table varchar_join1_vc2 ( + c1 int, + c2 varchar(20) +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@varchar_join1_vc2 +PREHOOK: query: create table varchar_join1_str ( + c1 int, + c2 string +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table varchar_join1_str ( + c1 int, + c2 string +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@varchar_join1_str +PREHOOK: query: load data local inpath '../data/files/vc1.txt' into table varchar_join1_vc1 +PREHOOK: type: LOAD +PREHOOK: Output: default@varchar_join1_vc1 +POSTHOOK: query: load data local inpath '../data/files/vc1.txt' into table varchar_join1_vc1 +POSTHOOK: type: LOAD +POSTHOOK: Output: default@varchar_join1_vc1 +PREHOOK: query: load data local inpath '../data/files/vc1.txt' into table varchar_join1_vc2 +PREHOOK: type: LOAD +PREHOOK: Output: default@varchar_join1_vc2 +POSTHOOK: query: load data local inpath '../data/files/vc1.txt' into table varchar_join1_vc2 +POSTHOOK: type: LOAD +POSTHOOK: Output: default@varchar_join1_vc2 +PREHOOK: query: load data local inpath '../data/files/vc1.txt' into table varchar_join1_str +PREHOOK: type: LOAD +PREHOOK: Output: default@varchar_join1_str +POSTHOOK: query: load data local inpath '../data/files/vc1.txt' into table varchar_join1_str +POSTHOOK: type: LOAD +POSTHOOK: Output: default@varchar_join1_str +PREHOOK: query: -- Join varchar with same length varchar +select * from varchar_join1_vc1 a join varchar_join1_vc1 b on (a.c2 = b.c2) order by a.c1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_join1_vc1 +#### A masked pattern was here #### +POSTHOOK: query: -- Join varchar with same length varchar +select * from varchar_join1_vc1 a join varchar_join1_vc1 b on (a.c2 = b.c2) order by a.c1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_join1_vc1 +#### A masked pattern was here #### +1 abc 1 abc +2 abc 2 abc +3 abc 3 abc +PREHOOK: query: -- Join varchar with different length varchar +select * from varchar_join1_vc1 a join varchar_join1_vc2 b on (a.c2 = b.c2) order by a.c1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_join1_vc1 +PREHOOK: Input: default@varchar_join1_vc2 +#### A masked pattern was here #### +POSTHOOK: query: -- Join varchar with different length varchar +select * from varchar_join1_vc1 a join varchar_join1_vc2 b on (a.c2 = b.c2) order by a.c1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_join1_vc1 +POSTHOOK: Input: default@varchar_join1_vc2 +#### A masked pattern was here #### +1 abc 1 abc +2 abc 2 abc +3 abc 3 abc +PREHOOK: query: -- Join varchar with string +select * from varchar_join1_vc1 a join varchar_join1_str b on (a.c2 = b.c2) order by a.c1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_join1_str +PREHOOK: Input: default@varchar_join1_vc1 +#### A masked pattern was here #### +POSTHOOK: query: -- Join varchar with string +select * from varchar_join1_vc1 a join varchar_join1_str b on (a.c2 = b.c2) order by a.c1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_join1_str +POSTHOOK: Input: default@varchar_join1_vc1 +#### A masked pattern was here #### +1 abc 1 abc +2 abc 2 abc +3 abc 3 abc +PREHOOK: query: drop table varchar_join1_vc1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@varchar_join1_vc1 +PREHOOK: Output: default@varchar_join1_vc1 +POSTHOOK: query: drop table varchar_join1_vc1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@varchar_join1_vc1 +POSTHOOK: Output: default@varchar_join1_vc1 +PREHOOK: query: drop table varchar_join1_vc2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@varchar_join1_vc2 +PREHOOK: Output: default@varchar_join1_vc2 +POSTHOOK: query: drop table varchar_join1_vc2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@varchar_join1_vc2 +POSTHOOK: Output: default@varchar_join1_vc2 +PREHOOK: query: drop table varchar_join1_str +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@varchar_join1_str +PREHOOK: Output: default@varchar_join1_str +POSTHOOK: query: drop table varchar_join1_str +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@varchar_join1_str +POSTHOOK: Output: default@varchar_join1_str diff --git ql/src/test/results/clientpositive/varchar_nested_types.q.out ql/src/test/results/clientpositive/varchar_nested_types.q.out new file mode 100644 index 0000000..19eb118 --- /dev/null +++ ql/src/test/results/clientpositive/varchar_nested_types.q.out @@ -0,0 +1,363 @@ +PREHOOK: query: drop table varchar_nested_1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table varchar_nested_1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table varchar_nested_array +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table varchar_nested_array +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table varchar_nested_map +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table varchar_nested_map +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table varchar_nested_struct +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table varchar_nested_struct +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table varchar_nested_cta +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table varchar_nested_cta +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table varchar_nested_view +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table varchar_nested_view +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table varchar_nested_1 (key int, value varchar(20)) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table varchar_nested_1 (key int, value varchar(20)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@varchar_nested_1 +PREHOOK: query: insert overwrite table varchar_nested_1 + select key, value from src limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@varchar_nested_1 +POSTHOOK: query: insert overwrite table varchar_nested_1 + select key, value from src limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@varchar_nested_1 +POSTHOOK: Lineage: varchar_nested_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- arrays +create table varchar_nested_array (c1 array) +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- arrays +create table varchar_nested_array (c1 array) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@varchar_nested_array +POSTHOOK: Lineage: varchar_nested_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table varchar_nested_array + select array(value, value) from varchar_nested_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_nested_1 +PREHOOK: Output: default@varchar_nested_array +POSTHOOK: query: insert overwrite table varchar_nested_array + select array(value, value) from varchar_nested_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_nested_1 +POSTHOOK: Output: default@varchar_nested_array +POSTHOOK: Lineage: varchar_nested_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_array.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +PREHOOK: query: describe varchar_nested_array +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe varchar_nested_array +POSTHOOK: type: DESCTABLE +POSTHOOK: Lineage: varchar_nested_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_array.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +c1 array None +PREHOOK: query: select * from varchar_nested_array +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_nested_array +#### A masked pattern was here #### +POSTHOOK: query: select * from varchar_nested_array +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_nested_array +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_nested_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_array.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +["val_238","val_238"] +["val_86","val_86"] +["val_311","val_311"] +["val_27","val_27"] +["val_165","val_165"] +PREHOOK: query: -- maps +create table varchar_nested_map (c1 map) +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- maps +create table varchar_nested_map (c1 map) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@varchar_nested_map +POSTHOOK: Lineage: varchar_nested_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_array.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +PREHOOK: query: insert overwrite table varchar_nested_map + select map(key, value) from varchar_nested_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_nested_1 +PREHOOK: Output: default@varchar_nested_map +POSTHOOK: query: insert overwrite table varchar_nested_map + select map(key, value) from varchar_nested_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_nested_1 +POSTHOOK: Output: default@varchar_nested_map +POSTHOOK: Lineage: varchar_nested_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_array.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_map.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +PREHOOK: query: describe varchar_nested_map +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe varchar_nested_map +POSTHOOK: type: DESCTABLE +POSTHOOK: Lineage: varchar_nested_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_array.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_map.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +c1 map None +PREHOOK: query: select * from varchar_nested_map +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_nested_map +#### A masked pattern was here #### +POSTHOOK: query: select * from varchar_nested_map +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_nested_map +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_nested_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_array.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_map.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +{238:"val_238"} +{86:"val_86"} +{311:"val_311"} +{27:"val_27"} +{165:"val_165"} +PREHOOK: query: -- structs +create table varchar_nested_struct (c1 struct) +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- structs +create table varchar_nested_struct (c1 struct) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@varchar_nested_struct +POSTHOOK: Lineage: varchar_nested_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_array.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_map.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +PREHOOK: query: insert overwrite table varchar_nested_struct + select named_struct('a', key, + 'b', value, + 'c', cast(value as string)) + from varchar_nested_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_nested_1 +PREHOOK: Output: default@varchar_nested_struct +POSTHOOK: query: insert overwrite table varchar_nested_struct + select named_struct('a', key, + 'b', value, + 'c', cast(value as string)) + from varchar_nested_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_nested_1 +POSTHOOK: Output: default@varchar_nested_struct +POSTHOOK: Lineage: varchar_nested_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_array.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_map.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_struct.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +PREHOOK: query: describe varchar_nested_struct +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe varchar_nested_struct +POSTHOOK: type: DESCTABLE +POSTHOOK: Lineage: varchar_nested_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_array.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_map.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_struct.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +c1 struct None +PREHOOK: query: select * from varchar_nested_struct +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_nested_struct +#### A masked pattern was here #### +POSTHOOK: query: select * from varchar_nested_struct +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_nested_struct +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_nested_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_array.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_map.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_struct.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +{"a":238,"b":"val_238","c":"val_238"} +{"a":86,"b":"val_86","c":"val_86"} +{"a":311,"b":"val_311","c":"val_311"} +{"a":27,"b":"val_27","c":"val_27"} +{"a":165,"b":"val_165","c":"val_165"} +PREHOOK: query: -- nested type with create table as +create table varchar_nested_cta as + select * from varchar_nested_struct +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@varchar_nested_struct +POSTHOOK: query: -- nested type with create table as +create table varchar_nested_cta as + select * from varchar_nested_struct +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@varchar_nested_struct +POSTHOOK: Output: default@varchar_nested_cta +POSTHOOK: Lineage: varchar_nested_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_array.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_map.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_struct.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +PREHOOK: query: describe varchar_nested_cta +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe varchar_nested_cta +POSTHOOK: type: DESCTABLE +POSTHOOK: Lineage: varchar_nested_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_array.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_map.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_struct.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +c1 struct None +PREHOOK: query: select * from varchar_nested_cta +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_nested_cta +#### A masked pattern was here #### +POSTHOOK: query: select * from varchar_nested_cta +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_nested_cta +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_nested_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_array.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_map.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_struct.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +{"a":238,"b":"val_238","c":"val_238"} +{"a":86,"b":"val_86","c":"val_86"} +{"a":311,"b":"val_311","c":"val_311"} +{"a":27,"b":"val_27","c":"val_27"} +{"a":165,"b":"val_165","c":"val_165"} +PREHOOK: query: -- nested type with view +create table varchar_nested_view as + select * from varchar_nested_struct +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@varchar_nested_struct +POSTHOOK: query: -- nested type with view +create table varchar_nested_view as + select * from varchar_nested_struct +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@varchar_nested_struct +POSTHOOK: Output: default@varchar_nested_view +POSTHOOK: Lineage: varchar_nested_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_array.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_map.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_struct.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +PREHOOK: query: describe varchar_nested_view +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe varchar_nested_view +POSTHOOK: type: DESCTABLE +POSTHOOK: Lineage: varchar_nested_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_array.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_map.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_struct.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +c1 struct None +PREHOOK: query: select * from varchar_nested_view +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_nested_view +#### A masked pattern was here #### +POSTHOOK: query: select * from varchar_nested_view +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_nested_view +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_nested_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_array.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_map.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_struct.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +{"a":238,"b":"val_238","c":"val_238"} +{"a":86,"b":"val_86","c":"val_86"} +{"a":311,"b":"val_311","c":"val_311"} +{"a":27,"b":"val_27","c":"val_27"} +{"a":165,"b":"val_165","c":"val_165"} +PREHOOK: query: drop table varchar_nested_1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@varchar_nested_1 +PREHOOK: Output: default@varchar_nested_1 +POSTHOOK: query: drop table varchar_nested_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@varchar_nested_1 +POSTHOOK: Output: default@varchar_nested_1 +POSTHOOK: Lineage: varchar_nested_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_array.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_map.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_struct.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +PREHOOK: query: drop table varchar_nested_array +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@varchar_nested_array +PREHOOK: Output: default@varchar_nested_array +POSTHOOK: query: drop table varchar_nested_array +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@varchar_nested_array +POSTHOOK: Output: default@varchar_nested_array +POSTHOOK: Lineage: varchar_nested_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_array.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_map.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_struct.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +PREHOOK: query: drop table varchar_nested_map +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@varchar_nested_map +PREHOOK: Output: default@varchar_nested_map +POSTHOOK: query: drop table varchar_nested_map +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@varchar_nested_map +POSTHOOK: Output: default@varchar_nested_map +POSTHOOK: Lineage: varchar_nested_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_array.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_map.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_struct.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +PREHOOK: query: drop table varchar_nested_struct +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@varchar_nested_struct +PREHOOK: Output: default@varchar_nested_struct +POSTHOOK: query: drop table varchar_nested_struct +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@varchar_nested_struct +POSTHOOK: Output: default@varchar_nested_struct +POSTHOOK: Lineage: varchar_nested_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_array.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_map.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_struct.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +PREHOOK: query: drop table varchar_nested_cta +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@varchar_nested_cta +PREHOOK: Output: default@varchar_nested_cta +POSTHOOK: query: drop table varchar_nested_cta +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@varchar_nested_cta +POSTHOOK: Output: default@varchar_nested_cta +POSTHOOK: Lineage: varchar_nested_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_array.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_map.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_struct.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +PREHOOK: query: drop table varchar_nested_view +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@varchar_nested_view +PREHOOK: Output: default@varchar_nested_view +POSTHOOK: query: drop table varchar_nested_view +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@varchar_nested_view +POSTHOOK: Output: default@varchar_nested_view +POSTHOOK: Lineage: varchar_nested_1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_1.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_nested_array.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_map.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] +POSTHOOK: Lineage: varchar_nested_struct.c1 EXPRESSION [(varchar_nested_1)varchar_nested_1.FieldSchema(name:key, type:int, comment:null), (varchar_nested_1)varchar_nested_1.FieldSchema(name:value, type:varchar(20), comment:null), ] diff --git ql/src/test/results/clientpositive/varchar_udf1.q.out ql/src/test/results/clientpositive/varchar_udf1.q.out new file mode 100644 index 0000000..03d2a96 --- /dev/null +++ ql/src/test/results/clientpositive/varchar_udf1.q.out @@ -0,0 +1,554 @@ +PREHOOK: query: drop table varchar_udf_1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table varchar_udf_1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table varchar_udf_1 (c1 string, c2 string, c3 varchar(10), c4 varchar(20)) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table varchar_udf_1 (c1 string, c2 string, c3 varchar(10), c4 varchar(20)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@varchar_udf_1 +PREHOOK: query: insert overwrite table varchar_udf_1 + select key, value, key, value from src limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@varchar_udf_1 +POSTHOOK: query: insert overwrite table varchar_udf_1 + select key, value, key, value from src limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@varchar_udf_1 +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- UDFs with varchar support +select + concat(c1, c2), + concat(c3, c4), + concat(c1, c2) = concat(c3, c4) +from varchar_udf_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: query: -- UDFs with varchar support +select + concat(c1, c2), + concat(c3, c4), + concat(c1, c2) = concat(c3, c4) +from varchar_udf_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +238val_238 238val_238 true +PREHOOK: query: select + upper(c2), + upper(c4), + upper(c2) = upper(c4) +from varchar_udf_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: query: select + upper(c2), + upper(c4), + upper(c2) = upper(c4) +from varchar_udf_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +VAL_238 VAL_238 true +PREHOOK: query: select + lower(c2), + lower(c4), + lower(c2) = lower(c4) +from varchar_udf_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: query: select + lower(c2), + lower(c4), + lower(c2) = lower(c4) +from varchar_udf_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +val_238 val_238 true +PREHOOK: query: -- Scalar UDFs +select + ascii(c2), + ascii(c4), + ascii(c2) = ascii(c4) +from varchar_udf_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: query: -- Scalar UDFs +select + ascii(c2), + ascii(c4), + ascii(c2) = ascii(c4) +from varchar_udf_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +118 118 true +PREHOOK: query: select + concat_ws('|', c1, c2), + concat_ws('|', c3, c4), + concat_ws('|', c1, c2) = concat_ws('|', c3, c4) +from varchar_udf_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: query: select + concat_ws('|', c1, c2), + concat_ws('|', c3, c4), + concat_ws('|', c1, c2) = concat_ws('|', c3, c4) +from varchar_udf_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +238|val_238 238|val_238 true +PREHOOK: query: select + decode(encode(c2, 'US-ASCII'), 'US-ASCII'), + decode(encode(c4, 'US-ASCII'), 'US-ASCII'), + decode(encode(c2, 'US-ASCII'), 'US-ASCII') = decode(encode(c4, 'US-ASCII'), 'US-ASCII') +from varchar_udf_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: query: select + decode(encode(c2, 'US-ASCII'), 'US-ASCII'), + decode(encode(c4, 'US-ASCII'), 'US-ASCII'), + decode(encode(c2, 'US-ASCII'), 'US-ASCII') = decode(encode(c4, 'US-ASCII'), 'US-ASCII') +from varchar_udf_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +val_238 val_238 true +PREHOOK: query: select + instr(c2, '_'), + instr(c4, '_'), + instr(c2, '_') = instr(c4, '_') +from varchar_udf_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: query: select + instr(c2, '_'), + instr(c4, '_'), + instr(c2, '_') = instr(c4, '_') +from varchar_udf_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +4 4 true +PREHOOK: query: select + length(c2), + length(c4), + length(c2) = length(c4) +from varchar_udf_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: query: select + length(c2), + length(c4), + length(c2) = length(c4) +from varchar_udf_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +7 7 true +PREHOOK: query: select + locate('a', 'abcdabcd', 3), + locate(cast('a' as varchar(1)), cast('abcdabcd' as varchar(10)), 3), + locate('a', 'abcdabcd', 3) = locate(cast('a' as varchar(1)), cast('abcdabcd' as varchar(10)), 3) +from varchar_udf_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: query: select + locate('a', 'abcdabcd', 3), + locate(cast('a' as varchar(1)), cast('abcdabcd' as varchar(10)), 3), + locate('a', 'abcdabcd', 3) = locate(cast('a' as varchar(1)), cast('abcdabcd' as varchar(10)), 3) +from varchar_udf_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +5 5 true +PREHOOK: query: select + lpad(c2, 15, ' '), + lpad(c4, 15, ' '), + lpad(c2, 15, ' ') = lpad(c4, 15, ' ') +from varchar_udf_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: query: select + lpad(c2, 15, ' '), + lpad(c4, 15, ' '), + lpad(c2, 15, ' ') = lpad(c4, 15, ' ') +from varchar_udf_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] + val_238 val_238 true +PREHOOK: query: select + ltrim(c2), + ltrim(c4), + ltrim(c2) = ltrim(c4) +from varchar_udf_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: query: select + ltrim(c2), + ltrim(c4), + ltrim(c2) = ltrim(c4) +from varchar_udf_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +val_238 val_238 true +PREHOOK: query: select + regexp(c2, 'val'), + regexp(c4, 'val'), + regexp(c2, 'val') = regexp(c4, 'val') +from varchar_udf_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: query: select + regexp(c2, 'val'), + regexp(c4, 'val'), + regexp(c2, 'val') = regexp(c4, 'val') +from varchar_udf_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +true true true +PREHOOK: query: select + regexp_extract(c2, 'val_([0-9]+)', 1), + regexp_extract(c4, 'val_([0-9]+)', 1), + regexp_extract(c2, 'val_([0-9]+)', 1) = regexp_extract(c4, 'val_([0-9]+)', 1) +from varchar_udf_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: query: select + regexp_extract(c2, 'val_([0-9]+)', 1), + regexp_extract(c4, 'val_([0-9]+)', 1), + regexp_extract(c2, 'val_([0-9]+)', 1) = regexp_extract(c4, 'val_([0-9]+)', 1) +from varchar_udf_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +238 238 true +PREHOOK: query: select + regexp_replace(c2, 'val', 'replaced'), + regexp_replace(c4, 'val', 'replaced'), + regexp_replace(c2, 'val', 'replaced') = regexp_replace(c4, 'val', 'replaced') +from varchar_udf_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: query: select + regexp_replace(c2, 'val', 'replaced'), + regexp_replace(c4, 'val', 'replaced'), + regexp_replace(c2, 'val', 'replaced') = regexp_replace(c4, 'val', 'replaced') +from varchar_udf_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +replaced_238 replaced_238 true +PREHOOK: query: select + reverse(c2), + reverse(c4), + reverse(c2) = reverse(c4) +from varchar_udf_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: query: select + reverse(c2), + reverse(c4), + reverse(c2) = reverse(c4) +from varchar_udf_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +832_lav 832_lav true +PREHOOK: query: select + rpad(c2, 15, ' '), + rpad(c4, 15, ' '), + rpad(c2, 15, ' ') = rpad(c4, 15, ' ') +from varchar_udf_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: query: select + rpad(c2, 15, ' '), + rpad(c4, 15, ' '), + rpad(c2, 15, ' ') = rpad(c4, 15, ' ') +from varchar_udf_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +val_238 val_238 true +PREHOOK: query: select + rtrim(c2), + rtrim(c4), + rtrim(c2) = rtrim(c4) +from varchar_udf_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: query: select + rtrim(c2), + rtrim(c4), + rtrim(c2) = rtrim(c4) +from varchar_udf_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +val_238 val_238 true +PREHOOK: query: select + sentences('See spot run. See jane run.'), + sentences(cast('See spot run. See jane run.' as varchar(50))) +from varchar_udf_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: query: select + sentences('See spot run. See jane run.'), + sentences(cast('See spot run. See jane run.' as varchar(50))) +from varchar_udf_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +[["See","spot","run"],["See","jane","run"]] [["See","spot","run"],["See","jane","run"]] +PREHOOK: query: select + split(c2, '_'), + split(c4, '_') +from varchar_udf_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: query: select + split(c2, '_'), + split(c4, '_') +from varchar_udf_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +["val","238"] ["val","238"] +PREHOOK: query: select + str_to_map('a:1,b:2,c:3',',',':'), + str_to_map(cast('a:1,b:2,c:3' as varchar(20)),',',':') +from varchar_udf_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: query: select + str_to_map('a:1,b:2,c:3',',',':'), + str_to_map(cast('a:1,b:2,c:3' as varchar(20)),',',':') +from varchar_udf_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +{"b":"2","a":"1","c":"3"} {"b":"2","a":"1","c":"3"} +PREHOOK: query: select + substr(c2, 1, 3), + substr(c4, 1, 3), + substr(c2, 1, 3) = substr(c4, 1, 3) +from varchar_udf_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: query: select + substr(c2, 1, 3), + substr(c4, 1, 3), + substr(c2, 1, 3) = substr(c4, 1, 3) +from varchar_udf_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +val val true +PREHOOK: query: select + trim(c2), + trim(c4), + trim(c2) = trim(c4) +from varchar_udf_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: query: select + trim(c2), + trim(c4), + trim(c2) = trim(c4) +from varchar_udf_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +val_238 val_238 true +PREHOOK: query: -- Aggregate Functions +select + compute_stats(c2, 16), + compute_stats(c4, 16) +from varchar_udf_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: query: -- Aggregate Functions +select + compute_stats(c2, 16), + compute_stats(c4, 16) +from varchar_udf_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +{"columntype":"String","maxlength":7,"avglength":6.6,"countnulls":0,"numdistinctvalues":7} {"columntype":"String","maxlength":7,"avglength":6.6,"countnulls":0,"numdistinctvalues":7} +PREHOOK: query: select + min(c2), + min(c4) +from varchar_udf_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: query: select + min(c2), + min(c4) +from varchar_udf_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +val_165 val_165 +PREHOOK: query: select + max(c2), + max(c4) +from varchar_udf_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: query: select + max(c2), + max(c4) +from varchar_udf_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_udf_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +val_86 val_86 +PREHOOK: query: drop table varchar_udf_1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@varchar_udf_1 +PREHOOK: Output: default@varchar_udf_1 +POSTHOOK: query: drop table varchar_udf_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@varchar_udf_1 +POSTHOOK: Output: default@varchar_udf_1 +POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] diff --git ql/src/test/results/clientpositive/varchar_union1.q.out ql/src/test/results/clientpositive/varchar_union1.q.out new file mode 100644 index 0000000..be409e6 --- /dev/null +++ ql/src/test/results/clientpositive/varchar_union1.q.out @@ -0,0 +1,157 @@ +PREHOOK: query: drop table varchar_union1_vc1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table varchar_union1_vc1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table varchar_union1_vc2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table varchar_union1_vc2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table varchar_union1_str +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table varchar_union1_str +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table varchar_union1_vc1 ( + c1 int, + c2 varchar(10) +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table varchar_union1_vc1 ( + c1 int, + c2 varchar(10) +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@varchar_union1_vc1 +PREHOOK: query: create table varchar_union1_vc2 ( + c1 int, + c2 varchar(20) +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table varchar_union1_vc2 ( + c1 int, + c2 varchar(20) +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@varchar_union1_vc2 +PREHOOK: query: create table varchar_union1_str ( + c1 int, + c2 string +) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table varchar_union1_str ( + c1 int, + c2 string +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@varchar_union1_str +PREHOOK: query: load data local inpath '../data/files/vc1.txt' into table varchar_union1_vc1 +PREHOOK: type: LOAD +PREHOOK: Output: default@varchar_union1_vc1 +POSTHOOK: query: load data local inpath '../data/files/vc1.txt' into table varchar_union1_vc1 +POSTHOOK: type: LOAD +POSTHOOK: Output: default@varchar_union1_vc1 +PREHOOK: query: load data local inpath '../data/files/vc1.txt' into table varchar_union1_vc2 +PREHOOK: type: LOAD +PREHOOK: Output: default@varchar_union1_vc2 +POSTHOOK: query: load data local inpath '../data/files/vc1.txt' into table varchar_union1_vc2 +POSTHOOK: type: LOAD +POSTHOOK: Output: default@varchar_union1_vc2 +PREHOOK: query: load data local inpath '../data/files/vc1.txt' into table varchar_union1_str +PREHOOK: type: LOAD +PREHOOK: Output: default@varchar_union1_str +POSTHOOK: query: load data local inpath '../data/files/vc1.txt' into table varchar_union1_str +POSTHOOK: type: LOAD +POSTHOOK: Output: default@varchar_union1_str +PREHOOK: query: -- union varchar with same length varchar +select * from ( + select * from varchar_union1_vc1 + union all + select * from varchar_union1_vc1 limit 1 +) q1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_union1_vc1 +#### A masked pattern was here #### +POSTHOOK: query: -- union varchar with same length varchar +select * from ( + select * from varchar_union1_vc1 + union all + select * from varchar_union1_vc1 limit 1 +) q1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_union1_vc1 +#### A masked pattern was here #### +1 abc +1 abc +2 abc +3 abc +PREHOOK: query: -- union varchar with different length varchar +select * from ( + select * from varchar_union1_vc1 + union all + select * from varchar_union1_vc2 limit 1 +) q1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_union1_vc1 +PREHOOK: Input: default@varchar_union1_vc2 +#### A masked pattern was here #### +POSTHOOK: query: -- union varchar with different length varchar +select * from ( + select * from varchar_union1_vc1 + union all + select * from varchar_union1_vc2 limit 1 +) q1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_union1_vc1 +POSTHOOK: Input: default@varchar_union1_vc2 +#### A masked pattern was here #### +1 abc +1 abc +2 abc +3 abc +PREHOOK: query: -- union varchar with string +select * from ( + select * from varchar_union1_vc1 + union all + select * from varchar_union1_str limit 1 +) q1 +PREHOOK: type: QUERY +PREHOOK: Input: default@varchar_union1_str +PREHOOK: Input: default@varchar_union1_vc1 +#### A masked pattern was here #### +POSTHOOK: query: -- union varchar with string +select * from ( + select * from varchar_union1_vc1 + union all + select * from varchar_union1_str limit 1 +) q1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@varchar_union1_str +POSTHOOK: Input: default@varchar_union1_vc1 +#### A masked pattern was here #### +1 abc +1 abc +2 abc +3 abc +PREHOOK: query: drop table varchar_union1_vc1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@varchar_union1_vc1 +PREHOOK: Output: default@varchar_union1_vc1 +POSTHOOK: query: drop table varchar_union1_vc1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@varchar_union1_vc1 +POSTHOOK: Output: default@varchar_union1_vc1 +PREHOOK: query: drop table varchar_union1_vc2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@varchar_union1_vc2 +PREHOOK: Output: default@varchar_union1_vc2 +POSTHOOK: query: drop table varchar_union1_vc2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@varchar_union1_vc2 +POSTHOOK: Output: default@varchar_union1_vc2 +PREHOOK: query: drop table varchar_union1_str +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@varchar_union1_str +PREHOOK: Output: default@varchar_union1_str +POSTHOOK: query: drop table varchar_union1_str +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@varchar_union1_str +POSTHOOK: Output: default@varchar_union1_str diff --git ql/src/test/results/compiler/plan/groupby2.q.xml ql/src/test/results/compiler/plan/groupby2.q.xml index 7258343..f6fb486 100755 --- ql/src/test/results/compiler/plan/groupby2.q.xml +++ ql/src/test/results/compiler/plan/groupby2.q.xml @@ -1491,17 +1491,7 @@ - - - org.apache.hadoop.hive.ql.udf.UDFConcat - - - org.apache.hadoop.hive.ql.udf.UDFConcat - - - concat - - + diff --git ql/src/test/results/compiler/plan/udf6.q.xml ql/src/test/results/compiler/plan/udf6.q.xml index 7d05dc4..d291202 100644 --- ql/src/test/results/compiler/plan/udf6.q.xml +++ ql/src/test/results/compiler/plan/udf6.q.xml @@ -379,17 +379,7 @@ - - - org.apache.hadoop.hive.ql.udf.UDFConcat - - - org.apache.hadoop.hive.ql.udf.UDFConcat - - - concat - - + diff --git serde/if/serde.thrift serde/if/serde.thrift index 9847720..f1a6802 100644 --- serde/if/serde.thrift +++ serde/if/serde.thrift @@ -50,6 +50,8 @@ const string BIGINT_TYPE_NAME = "bigint"; const string FLOAT_TYPE_NAME = "float"; const string DOUBLE_TYPE_NAME = "double"; const string STRING_TYPE_NAME = "string"; +const string CHAR_TYPE_NAME = "char"; +const string VARCHAR_TYPE_NAME = "varchar"; const string DATE_TYPE_NAME = "date"; const string DATETIME_TYPE_NAME = "datetime"; const string TIMESTAMP_TYPE_NAME = "timestamp"; @@ -64,7 +66,7 @@ const string UNION_TYPE_NAME = "uniontype"; const string LIST_COLUMNS = "columns"; const string LIST_COLUMN_TYPES = "columns.types"; -const set PrimitiveTypes = [ VOID_TYPE_NAME BOOLEAN_TYPE_NAME TINYINT_TYPE_NAME SMALLINT_TYPE_NAME INT_TYPE_NAME BIGINT_TYPE_NAME FLOAT_TYPE_NAME DOUBLE_TYPE_NAME STRING_TYPE_NAME DATE_TYPE_NAME DATETIME_TYPE_NAME TIMESTAMP_TYPE_NAME DECIMAL_TYPE_NAME BINARY_TYPE_NAME], +const set PrimitiveTypes = [ VOID_TYPE_NAME BOOLEAN_TYPE_NAME TINYINT_TYPE_NAME SMALLINT_TYPE_NAME INT_TYPE_NAME BIGINT_TYPE_NAME FLOAT_TYPE_NAME DOUBLE_TYPE_NAME STRING_TYPE_NAME VARCHAR_TYPE_NAME CHAR_TYPE_NAME DATE_TYPE_NAME DATETIME_TYPE_NAME TIMESTAMP_TYPE_NAME DECIMAL_TYPE_NAME BINARY_TYPE_NAME], const set CollectionTypes = [ LIST_TYPE_NAME MAP_TYPE_NAME ], diff --git serde/src/gen/thrift/gen-cpp/serde_constants.cpp serde/src/gen/thrift/gen-cpp/serde_constants.cpp index 3997026..86a24af 100644 --- serde/src/gen/thrift/gen-cpp/serde_constants.cpp +++ serde/src/gen/thrift/gen-cpp/serde_constants.cpp @@ -57,6 +57,10 @@ serdeConstants::serdeConstants() { STRING_TYPE_NAME = "string"; + CHAR_TYPE_NAME = "char"; + + VARCHAR_TYPE_NAME = "varchar"; + DATE_TYPE_NAME = "date"; DATETIME_TYPE_NAME = "datetime"; @@ -88,6 +92,8 @@ serdeConstants::serdeConstants() { PrimitiveTypes.insert("float"); PrimitiveTypes.insert("double"); PrimitiveTypes.insert("string"); + PrimitiveTypes.insert("varchar"); + PrimitiveTypes.insert("char"); PrimitiveTypes.insert("date"); PrimitiveTypes.insert("datetime"); PrimitiveTypes.insert("timestamp"); diff --git serde/src/gen/thrift/gen-cpp/serde_constants.h serde/src/gen/thrift/gen-cpp/serde_constants.h index 0a63308..117d6c6 100644 --- serde/src/gen/thrift/gen-cpp/serde_constants.h +++ serde/src/gen/thrift/gen-cpp/serde_constants.h @@ -38,6 +38,8 @@ class serdeConstants { std::string FLOAT_TYPE_NAME; std::string DOUBLE_TYPE_NAME; std::string STRING_TYPE_NAME; + std::string CHAR_TYPE_NAME; + std::string VARCHAR_TYPE_NAME; std::string DATE_TYPE_NAME; std::string DATETIME_TYPE_NAME; std::string TIMESTAMP_TYPE_NAME; diff --git serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java index 28f8d6a..096f881 100644 --- serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java +++ serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java @@ -79,6 +79,10 @@ public static final String STRING_TYPE_NAME = "string"; + public static final String CHAR_TYPE_NAME = "char"; + + public static final String VARCHAR_TYPE_NAME = "varchar"; + public static final String DATE_TYPE_NAME = "date"; public static final String DATETIME_TYPE_NAME = "datetime"; @@ -112,6 +116,8 @@ PrimitiveTypes.add("float"); PrimitiveTypes.add("double"); PrimitiveTypes.add("string"); + PrimitiveTypes.add("varchar"); + PrimitiveTypes.add("char"); PrimitiveTypes.add("date"); PrimitiveTypes.add("datetime"); PrimitiveTypes.add("timestamp"); diff --git serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/test/ThriftTestObj.java serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/test/ThriftTestObj.java index 1b708dd..dda3c5f 100644 --- serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/test/ThriftTestObj.java +++ serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/test/ThriftTestObj.java @@ -528,7 +528,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, ThriftTestObj struc struct.field3 = new ArrayList(_list0.size); for (int _i1 = 0; _i1 < _list0.size; ++_i1) { - InnerStruct _elem2; // required + InnerStruct _elem2; // optional _elem2 = new InnerStruct(); _elem2.read(iprot); struct.field3.add(_elem2); @@ -636,7 +636,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, ThriftTestObj struct struct.field3 = new ArrayList(_list5.size); for (int _i6 = 0; _i6 < _list5.size; ++_i6) { - InnerStruct _elem7; // required + InnerStruct _elem7; // optional _elem7 = new InnerStruct(); _elem7.read(iprot); struct.field3.add(_elem7); diff --git serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/Complex.java serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/Complex.java index e36a792..aa404bf 100644 --- serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/Complex.java +++ serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/Complex.java @@ -836,7 +836,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Complex struct) thr struct.lint = new ArrayList(_list0.size); for (int _i1 = 0; _i1 < _list0.size; ++_i1) { - int _elem2; // required + int _elem2; // optional _elem2 = iprot.readI32(); struct.lint.add(_elem2); } @@ -854,7 +854,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Complex struct) thr struct.lString = new ArrayList(_list3.size); for (int _i4 = 0; _i4 < _list3.size; ++_i4) { - String _elem5; // required + String _elem5; // optional _elem5 = iprot.readString(); struct.lString.add(_elem5); } @@ -872,7 +872,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Complex struct) thr struct.lintString = new ArrayList(_list6.size); for (int _i7 = 0; _i7 < _list6.size; ++_i7) { - IntString _elem8; // required + IntString _elem8; // optional _elem8 = new IntString(); _elem8.read(iprot); struct.lintString.add(_elem8); @@ -1074,7 +1074,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, Complex struct) thro struct.lint = new ArrayList(_list21.size); for (int _i22 = 0; _i22 < _list21.size; ++_i22) { - int _elem23; // required + int _elem23; // optional _elem23 = iprot.readI32(); struct.lint.add(_elem23); } @@ -1087,7 +1087,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, Complex struct) thro struct.lString = new ArrayList(_list24.size); for (int _i25 = 0; _i25 < _list24.size; ++_i25) { - String _elem26; // required + String _elem26; // optional _elem26 = iprot.readString(); struct.lString.add(_elem26); } @@ -1100,7 +1100,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, Complex struct) thro struct.lintString = new ArrayList(_list27.size); for (int _i28 = 0; _i28 < _list27.size; ++_i28) { - IntString _elem29; // required + IntString _elem29; // optional _elem29 = new IntString(); _elem29.read(iprot); struct.lintString.add(_elem29); diff --git serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/MegaStruct.java serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/MegaStruct.java index 386fef9..fba49e4 100644 --- serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/MegaStruct.java +++ serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/MegaStruct.java @@ -2280,7 +2280,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, MegaStruct struct) _val19 = new ArrayList(_list20.size); for (int _i21 = 0; _i21 < _list20.size; ++_i21) { - String _elem22; // required + String _elem22; // optional _elem22 = iprot.readString(); _val19.add(_elem22); } @@ -2310,7 +2310,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, MegaStruct struct) _val26 = new ArrayList(_list27.size); for (int _i28 = 0; _i28 < _list27.size; ++_i28) { - MiniStruct _elem29; // required + MiniStruct _elem29; // optional _elem29 = new MiniStruct(); _elem29.read(iprot); _val26.add(_elem29); @@ -2333,7 +2333,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, MegaStruct struct) struct.my_stringlist = new ArrayList(_list30.size); for (int _i31 = 0; _i31 < _list30.size; ++_i31) { - String _elem32; // required + String _elem32; // optional _elem32 = iprot.readString(); struct.my_stringlist.add(_elem32); } @@ -2351,7 +2351,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, MegaStruct struct) struct.my_structlist = new ArrayList(_list33.size); for (int _i34 = 0; _i34 < _list33.size; ++_i34) { - MiniStruct _elem35; // required + MiniStruct _elem35; // optional _elem35 = new MiniStruct(); _elem35.read(iprot); struct.my_structlist.add(_elem35); @@ -2370,7 +2370,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, MegaStruct struct) struct.my_enumlist = new ArrayList(_list36.size); for (int _i37 = 0; _i37 < _list36.size; ++_i37) { - MyEnum _elem38; // required + MyEnum _elem38; // optional _elem38 = MyEnum.findByValue(iprot.readI32()); struct.my_enumlist.add(_elem38); } @@ -2388,7 +2388,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, MegaStruct struct) struct.my_stringset = new HashSet(2*_set39.size); for (int _i40 = 0; _i40 < _set39.size; ++_i40) { - String _elem41; // required + String _elem41; // optional _elem41 = iprot.readString(); struct.my_stringset.add(_elem41); } @@ -2406,7 +2406,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, MegaStruct struct) struct.my_enumset = new HashSet(2*_set42.size); for (int _i43 = 0; _i43 < _set42.size; ++_i43) { - MyEnum _elem44; // required + MyEnum _elem44; // optional _elem44 = MyEnum.findByValue(iprot.readI32()); struct.my_enumset.add(_elem44); } @@ -2424,7 +2424,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, MegaStruct struct) struct.my_structset = new HashSet(2*_set45.size); for (int _i46 = 0; _i46 < _set45.size; ++_i46) { - MiniStruct _elem47; // required + MiniStruct _elem47; // optional _elem47 = new MiniStruct(); _elem47.read(iprot); struct.my_structset.add(_elem47); @@ -3023,7 +3023,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, MegaStruct struct) t _val95 = new ArrayList(_list96.size); for (int _i97 = 0; _i97 < _list96.size; ++_i97) { - String _elem98; // required + String _elem98; // optional _elem98 = iprot.readString(); _val95.add(_elem98); } @@ -3047,7 +3047,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, MegaStruct struct) t _val102 = new ArrayList(_list103.size); for (int _i104 = 0; _i104 < _list103.size; ++_i104) { - MiniStruct _elem105; // required + MiniStruct _elem105; // optional _elem105 = new MiniStruct(); _elem105.read(iprot); _val102.add(_elem105); @@ -3064,7 +3064,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, MegaStruct struct) t struct.my_stringlist = new ArrayList(_list106.size); for (int _i107 = 0; _i107 < _list106.size; ++_i107) { - String _elem108; // required + String _elem108; // optional _elem108 = iprot.readString(); struct.my_stringlist.add(_elem108); } @@ -3077,7 +3077,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, MegaStruct struct) t struct.my_structlist = new ArrayList(_list109.size); for (int _i110 = 0; _i110 < _list109.size; ++_i110) { - MiniStruct _elem111; // required + MiniStruct _elem111; // optional _elem111 = new MiniStruct(); _elem111.read(iprot); struct.my_structlist.add(_elem111); @@ -3091,7 +3091,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, MegaStruct struct) t struct.my_enumlist = new ArrayList(_list112.size); for (int _i113 = 0; _i113 < _list112.size; ++_i113) { - MyEnum _elem114; // required + MyEnum _elem114; // optional _elem114 = MyEnum.findByValue(iprot.readI32()); struct.my_enumlist.add(_elem114); } @@ -3104,7 +3104,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, MegaStruct struct) t struct.my_stringset = new HashSet(2*_set115.size); for (int _i116 = 0; _i116 < _set115.size; ++_i116) { - String _elem117; // required + String _elem117; // optional _elem117 = iprot.readString(); struct.my_stringset.add(_elem117); } @@ -3117,7 +3117,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, MegaStruct struct) t struct.my_enumset = new HashSet(2*_set118.size); for (int _i119 = 0; _i119 < _set118.size; ++_i119) { - MyEnum _elem120; // required + MyEnum _elem120; // optional _elem120 = MyEnum.findByValue(iprot.readI32()); struct.my_enumset.add(_elem120); } @@ -3130,7 +3130,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, MegaStruct struct) t struct.my_structset = new HashSet(2*_set121.size); for (int _i122 = 0; _i122 < _set121.size; ++_i122) { - MiniStruct _elem123; // required + MiniStruct _elem123; // optional _elem123 = new MiniStruct(); _elem123.read(iprot); struct.my_structset.add(_elem123); diff --git serde/src/gen/thrift/gen-php/org/apache/hadoop/hive/serde/Types.php serde/src/gen/thrift/gen-php/org/apache/hadoop/hive/serde/Types.php index 130c17e..8a0b415 100644 --- serde/src/gen/thrift/gen-php/org/apache/hadoop/hive/serde/Types.php +++ serde/src/gen/thrift/gen-php/org/apache/hadoop/hive/serde/Types.php @@ -62,6 +62,10 @@ $GLOBALS['serde_CONSTANTS']['DOUBLE_TYPE_NAME'] = "double"; $GLOBALS['serde_CONSTANTS']['STRING_TYPE_NAME'] = "string"; +$GLOBALS['serde_CONSTANTS']['CHAR_TYPE_NAME'] = "char"; + +$GLOBALS['serde_CONSTANTS']['VARCHAR_TYPE_NAME'] = "varchar"; + $GLOBALS['serde_CONSTANTS']['DATE_TYPE_NAME'] = "date"; $GLOBALS['serde_CONSTANTS']['DATETIME_TYPE_NAME'] = "datetime"; @@ -94,6 +98,8 @@ $GLOBALS['serde_CONSTANTS']['PrimitiveTypes'] = array( "float" => true, "double" => true, "string" => true, + "varchar" => true, + "char" => true, "date" => true, "datetime" => true, "timestamp" => true, diff --git serde/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/constants.py serde/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/constants.py index 623bf0e..47aab22 100644 --- serde/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/constants.py +++ serde/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/constants.py @@ -32,6 +32,8 @@ FLOAT_TYPE_NAME = "float" DOUBLE_TYPE_NAME = "double" STRING_TYPE_NAME = "string" +CHAR_TYPE_NAME = "char" +VARCHAR_TYPE_NAME = "varchar" DATE_TYPE_NAME = "date" DATETIME_TYPE_NAME = "datetime" TIMESTAMP_TYPE_NAME = "timestamp" @@ -53,6 +55,8 @@ "float", "double", "string", + "varchar", + "char", "date", "datetime", "timestamp", diff --git serde/src/gen/thrift/gen-rb/serde_constants.rb serde/src/gen/thrift/gen-rb/serde_constants.rb index bd17761..200cefd 100644 --- serde/src/gen/thrift/gen-rb/serde_constants.rb +++ serde/src/gen/thrift/gen-rb/serde_constants.rb @@ -53,6 +53,10 @@ DOUBLE_TYPE_NAME = %q"double" STRING_TYPE_NAME = %q"string" +CHAR_TYPE_NAME = %q"char" + +VARCHAR_TYPE_NAME = %q"varchar" + DATE_TYPE_NAME = %q"date" DATETIME_TYPE_NAME = %q"datetime" @@ -85,6 +89,8 @@ PrimitiveTypes = Set.new([ %q"float", %q"double", %q"string", + %q"varchar", + %q"char", %q"date", %q"datetime", %q"timestamp", diff --git serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java index f88fd94..c06218f 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java @@ -40,6 +40,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; @@ -284,6 +285,13 @@ static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi, Stri sb.append('"'); break; } + case VARCHAR: { + sb.append('"'); + sb.append(escapeString(((HiveVarcharObjectInspector) poi) + .getPrimitiveJavaObject(o).toString())); + sb.append('"'); + break; + } case DATE: { sb.append('"'); sb.append(((DateObjectInspector) poi) diff --git serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroObjectInspectorGenerator.java serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroObjectInspectorGenerator.java index feef48a..111a175 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroObjectInspectorGenerator.java +++ serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroObjectInspectorGenerator.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.ParameterizedPrimitiveTypeUtils; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -92,7 +93,8 @@ private ObjectInspector createObjectInspectorWorker(TypeInfo ti) throws SerDeExc case PRIMITIVE: PrimitiveTypeInfo pti = (PrimitiveTypeInfo)ti; result = PrimitiveObjectInspectorFactory - .getPrimitiveJavaObjectInspector(pti.getPrimitiveCategory()); + .getPrimitiveJavaObjectInspector(pti.getPrimitiveCategory(), + ParameterizedPrimitiveTypeUtils.getTypeParamsFromTypeInfo(pti)); break; case STRUCT: StructTypeInfo sti = (StructTypeInfo)ti; diff --git serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java index 0141198..c379ec2 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java +++ serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java @@ -20,6 +20,8 @@ import java.io.IOException; import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; @@ -31,6 +33,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.AbstractSerDe; @@ -39,6 +42,7 @@ import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; @@ -56,14 +60,18 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.ParameterizedPrimitiveTypeUtils.HiveVarcharSerDeHelper; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeParams; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.ParameterizedPrimitiveTypeUtils; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -276,48 +284,18 @@ static Object deserialize(InputByteBuffer buffer, TypeInfo type, } case STRING: { Text r = reuse == null ? new Text() : (Text) reuse; - // Get the actual length first - int start = buffer.tell(); - int length = 0; - do { - byte b = buffer.read(invert); - if (b == 0) { - // end of string - break; - } - if (b == 1) { - // the last char is an escape char. read the actual char - buffer.read(invert); - } - length++; - } while (true); + return deserializeText(buffer, invert, r); + } - if (length == buffer.tell() - start) { - // No escaping happened, so we are already done. - r.set(buffer.getData(), start, length); - } else { - // Escaping happened, we need to copy byte-by-byte. - // 1. Set the length first. - r.set(buffer.getData(), start, length); - // 2. Reset the pointer. - buffer.seek(start); - // 3. Copy the data. - byte[] rdata = r.getBytes(); - for (int i = 0; i < length; i++) { - byte b = buffer.read(invert); - if (b == 1) { - // The last char is an escape char, read the actual char. - // The serialization format escape \0 to \1, and \1 to \2, - // to make sure the string is null-terminated. - b = (byte) (buffer.read(invert) - 1); - } - rdata[i] = b; - } - // 4. Read the null terminator. - byte b = buffer.read(invert); - assert (b == 0); - } - return r; + case VARCHAR: { + HiveVarcharWritable r = + reuse == null ? new HiveVarcharWritable() : (HiveVarcharWritable) reuse; + // Use HiveVarchar's internal Text member to read the value. + deserializeText(buffer, invert, r.getTextValue()); + // If we cache helper data for deserialization we could avoid having + // to call getVarcharMaxLength() on every deserialize call. + r.enforceMaxLength(getVarcharMaxLength(type)); + return r; } case BINARY: { @@ -552,6 +530,60 @@ private static int deserializeInt(InputByteBuffer buffer, boolean invert) throws return v; } + static int getVarcharMaxLength(TypeInfo type) { + VarcharTypeParams typeParams = (VarcharTypeParams) ((PrimitiveTypeInfo) type).getParameters(); + if (typeParams != null ) { + return typeParams.length; + } + return -1; + } + + static Text deserializeText(InputByteBuffer buffer, boolean invert, Text r) + throws IOException { + // Get the actual length first + int start = buffer.tell(); + int length = 0; + do { + byte b = buffer.read(invert); + if (b == 0) { + // end of string + break; + } + if (b == 1) { + // the last char is an escape char. read the actual char + buffer.read(invert); + } + length++; + } while (true); + + if (length == buffer.tell() - start) { + // No escaping happened, so we are already done. + r.set(buffer.getData(), start, length); + } else { + // Escaping happened, we need to copy byte-by-byte. + // 1. Set the length first. + r.set(buffer.getData(), start, length); + // 2. Reset the pointer. + buffer.seek(start); + // 3. Copy the data. + byte[] rdata = r.getBytes(); + for (int i = 0; i < length; i++) { + byte b = buffer.read(invert); + if (b == 1) { + // The last char is an escape char, read the actual char. + // The serialization format escape \0 to \1, and \1 to \2, + // to make sure the string is null-terminated. + b = (byte) (buffer.read(invert) - 1); + } + rdata[i] = b; + } + // 4. Read the null terminator. + byte b = buffer.read(invert); + assert (b == 0); + } + return r; + } + BytesWritable serializeBytesWritable = new BytesWritable(); OutputByteBuffer outputByteBuffer = new OutputByteBuffer(); @@ -572,7 +604,7 @@ public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDe } static void serialize(OutputByteBuffer buffer, Object o, ObjectInspector oi, - boolean invert) { + boolean invert) throws SerDeException { // Is this field a null? if (o == null) { buffer.write((byte) 0, invert); @@ -668,6 +700,18 @@ static void serialize(OutputByteBuffer buffer, Object o, ObjectInspector oi, return; } + case VARCHAR: { + HiveVarcharObjectInspector hcoi = (HiveVarcharObjectInspector)poi; + HiveVarcharWritable hc = hcoi.getPrimitiveWritableObject(o); + try { + ByteBuffer bb = Text.encode(hc.getHiveVarchar().getValue()); + serializeBytes(buffer, bb.array(), bb.limit(), invert); + } catch (CharacterCodingException err) { + throw new SerDeException(err); + } + return; + } + case BINARY: { BinaryObjectInspector baoi = (BinaryObjectInspector) poi; BytesWritable ba = baoi.getPrimitiveWritableObject(o); diff --git serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java index 21cf616..aa5fbaa 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java @@ -35,8 +35,10 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; import org.apache.hadoop.hive.serde2.thrift.ConfigurableTProtocol; import org.apache.hadoop.hive.serde2.thrift.TReflectionUtils; +import org.apache.hadoop.hive.serde2.typeinfo.ParameterizedPrimitiveTypeUtils; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; @@ -173,9 +175,11 @@ public static ObjectInspector dynamicSerDeStructBaseToObjectInspector( dynamicSerDeStructBaseToObjectInspector(btMap.getKeyType()), dynamicSerDeStructBaseToObjectInspector(btMap.getValueType())); } else if (bt.isPrimitive()) { + PrimitiveTypeEntry pte = PrimitiveObjectInspectorUtils + .getTypeEntryFromPrimitiveJavaClass(bt.getRealType()); return PrimitiveObjectInspectorFactory - .getPrimitiveJavaObjectInspector(PrimitiveObjectInspectorUtils - .getTypeEntryFromPrimitiveJavaClass(bt.getRealType()).primitiveCategory); + .getPrimitiveJavaObjectInspector(pte.primitiveCategory, + ParameterizedPrimitiveTypeUtils.getTypeParamsFromPrimitiveTypeEntry(pte)); } else { // Must be a struct DynamicSerDeStructBase btStruct = (DynamicSerDeStructBase) bt; diff --git serde/src/java/org/apache/hadoop/hive/serde2/io/DoubleWritable.java serde/src/java/org/apache/hadoop/hive/serde2/io/DoubleWritable.java index 8ebc785..f3a2b1a 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/io/DoubleWritable.java +++ serde/src/java/org/apache/hadoop/hive/serde2/io/DoubleWritable.java @@ -17,7 +17,7 @@ */ /** - * This file is back-ported from hadoop-0.19, to make sure hive can run + * This file is back-ported from hadoop-0.19, to make sure hive can run * with hadoop-0.17. */ package org.apache.hadoop.hive.serde2.io; diff --git serde/src/java/org/apache/hadoop/hive/serde2/io/HiveVarcharWritable.java serde/src/java/org/apache/hadoop/hive/serde2/io/HiveVarcharWritable.java new file mode 100644 index 0000000..aaaee10 --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/io/HiveVarcharWritable.java @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.io; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.hive.common.type.HiveBaseChar; +import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.shims.ShimLoader; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableComparable; + +public class HiveVarcharWritable implements WritableComparable{ + protected Text value = new Text(); + transient protected int characterLength = -1; + + public HiveVarcharWritable() { + } + + public HiveVarcharWritable(HiveVarchar hc) { + set(hc); + } + + public HiveVarcharWritable(HiveVarcharWritable hcw) { + set(hcw); + } + + public void set(HiveVarchar val) { + set(val.getValue()); + } + + public void set(String val) { + set(val, -1); // copy entire string value + } + + public void set(HiveVarcharWritable val) { + value.set(val.value); + characterLength = val.characterLength; + } + + public void set(HiveVarcharWritable val, int maxLength) { + if (val.characterLength > 0 && val.characterLength >= maxLength) { + set(val); + } else { + set(val.getHiveVarchar(), maxLength); + } + } + + public void set(HiveVarchar val, int len) { + set(val.getValue(), len); + } + + public void set(String val, int maxLength) { + value.set(HiveBaseChar.enforceMaxLength(val, maxLength)); + } + + public HiveVarchar getHiveVarchar() { + return new HiveVarchar(value.toString(), -1); + } + + public int getCharacterLength() { + if (characterLength < 0) { + characterLength = getHiveVarchar().getCharacterLength(); + } + return characterLength; + } + + public void enforceMaxLength(int maxLength) { + // Might be possible to truncate the existing Text value, for now just do something simple. + set(getHiveVarchar(), maxLength); + } + + public void readFields(DataInput in) throws IOException { + value.readFields(in); + } + + public void write(DataOutput out) throws IOException { + value.write(out); + } + + public int compareTo(HiveVarcharWritable rhs) { + return ShimLoader.getHadoopShims().compareText(value, rhs.value); + } + + public boolean equals(Object obj) { + if (obj == null || !(obj instanceof HiveVarcharWritable)) { + return false; + } + return value.equals(((HiveVarcharWritable)obj).value); + } + + @Override + public String toString() { + return value.toString(); + } + + public int hashCode() { + return value.hashCode(); + } + + /** + * Access to the internal Text member. Use with care. + * @return + */ + public Text getTextValue() { + return value; + } +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java index a08b4a8..a5bfc97 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyByteObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyDoubleObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyFloatObjectInspector; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyHiveVarcharObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyIntObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyLongObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyPrimitiveObjectInspectorFactory; @@ -53,6 +54,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.typeinfo.BaseTypeParams; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; @@ -112,6 +114,8 @@ return new LazyDouble((LazyDoubleObjectInspector) oi); case STRING: return new LazyString((LazyStringObjectInspector) oi); + case VARCHAR: + return new LazyHiveVarchar((LazyHiveVarcharObjectInspector) oi); case DATE: return new LazyDate((LazyDateObjectInspector) oi); case TIMESTAMP: @@ -215,8 +219,9 @@ public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, ObjectInspector.Category c = typeInfo.getCategory(); switch (c) { case PRIMITIVE: + BaseTypeParams typeParams = ((PrimitiveTypeInfo)typeInfo).getParameters(); return LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector( - ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory(), escaped, + ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory(), typeParams, escaped, escapeChar); case MAP: return LazyObjectInspectorFactory.getLazySimpleMapObjectInspector( diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java new file mode 100644 index 0000000..1286cba --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.lazy; + +import java.nio.charset.CharacterCodingException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyHiveVarcharObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeParams; +import org.apache.hadoop.io.Text; + +/** + * LazyObject for storing a value of HiveVarchar. + * + */ +public class LazyHiveVarchar extends + LazyPrimitive { + + private static final Log LOG = LogFactory.getLog(LazyHiveVarchar.class); + + protected int maxLength = -1; + + public LazyHiveVarchar(LazyHiveVarcharObjectInspector oi) { + super(oi); + VarcharTypeParams typeParams = (VarcharTypeParams)oi.getTypeParams(); + if (typeParams == null) { + throw new RuntimeException("varchar type used without type params"); + } + maxLength = typeParams.getLength(); + data = new HiveVarcharWritable(); + } + + public LazyHiveVarchar(LazyHiveVarchar copy) { + super(copy); + this.maxLength = copy.maxLength; + data = new HiveVarcharWritable(copy.data); + } + + public void setValue(LazyHiveVarchar copy) { + data.set(copy.data, maxLength); + } + + @Override + public void init(ByteArrayRef bytes, int start, int length) { + String byteData = null; + try { + byteData = Text.decode(bytes.getData(), start, length); + data.set(byteData, maxLength); + isNull = false; + } catch (CharacterCodingException e) { + isNull = true; + LOG.debug("Data not in the HiveVarchar data type range so converted to null.", e); + } + } + +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java index 8a538c0..05822cb 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; @@ -39,6 +40,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; @@ -225,6 +227,12 @@ public static void writePrimitiveUTF8(OutputStream out, Object o, break; } + case VARCHAR: { + HiveVarcharWritable hc = ((HiveVarcharObjectInspector)oi).getPrimitiveWritableObject(o); + ByteBuffer b = Text.encode(hc.toString()); + writeEscaped(out, b.array(), 0, b.limit(), escaped, escapeChar, needsEscape); + break; + } case BINARY: { BytesWritable bw = ((BinaryObjectInspector) oi).getPrimitiveWritableObject(o); byte[] toEncode = new byte[bw.getLength()]; diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java new file mode 100644 index 0000000..3559c5e --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive; + + +import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; +import org.apache.hadoop.hive.serde2.lazy.LazyHiveVarchar; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; +import org.apache.hadoop.hive.serde2.typeinfo.BaseTypeParams; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeParams; +import org.apache.hadoop.hive.serde2.typeinfo.ParameterizedPrimitiveTypeUtils; + +public class LazyHiveVarcharObjectInspector + extends AbstractPrimitiveLazyObjectInspector + implements HiveVarcharObjectInspector { + + protected LazyHiveVarcharObjectInspector(PrimitiveTypeEntry typeEntry) { + super(typeEntry); + if (typeEntry.primitiveCategory != PrimitiveCategory.VARCHAR) { + throw new RuntimeException( + "TypeEntry of type varchar expected, got " + typeEntry.primitiveCategory); + } + } + + @Override + public Object copyObject(Object o) { + if (o == null) { + return null; + } + + LazyHiveVarchar ret = new LazyHiveVarchar(this); + ret.setValue((LazyHiveVarchar) o); + return ret; + } + + @Override + public HiveVarchar getPrimitiveJavaObject(Object o) { + if (o == null) { + return null; + } + + HiveVarchar ret = ((LazyHiveVarchar) o).getWritableObject().getHiveVarchar(); + if (!ParameterizedPrimitiveTypeUtils.doesPrimitiveMatchTypeParams( + ret, (VarcharTypeParams) typeParams)) { + HiveVarchar newValue = new HiveVarchar(ret, ((VarcharTypeParams) typeParams).length); + return newValue; + } + return ret; + } + + public String toString() { + return getTypeName(); + } +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java index afbf454..c124c89 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java @@ -21,7 +21,12 @@ import java.util.ArrayList; import java.util.HashMap; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; +import org.apache.hadoop.hive.serde2.typeinfo.BaseTypeParams; /** * LazyPrimitiveObjectInspectorFactory is the primary way to create new @@ -61,6 +66,8 @@ new LazyBinaryObjectInspector(); public static final LazyHiveDecimalObjectInspector LAZY_BIG_DECIMAL_OBJECT_INSPECTOR = new LazyHiveDecimalObjectInspector(); + public static final LazyHiveVarcharObjectInspector LAZY_VARCHAR_OBJECT_INSPECTOR = + new LazyHiveVarcharObjectInspector(PrimitiveObjectInspectorUtils.varcharTypeEntry); static HashMap, LazyStringObjectInspector> cachedLazyStringObjectInspector = new HashMap, LazyStringObjectInspector>(); @@ -79,9 +86,37 @@ public static LazyStringObjectInspector getLazyStringObjectInspector( return result; } + static PrimitiveObjectInspectorUtils.ParameterizedObjectInspectorMap + cachedParameterizedLazyObjectInspectors = + new PrimitiveObjectInspectorUtils.ParameterizedObjectInspectorMap(); + + public static PrimitiveObjectInspector getParameterizedObjectInspector( + PrimitiveCategory primitiveCategory, + BaseTypeParams typeParams) { + PrimitiveObjectInspector poi = + cachedParameterizedLazyObjectInspectors.getObjectInspector(primitiveCategory, typeParams); + if (poi == null) { + // Object inspector hasn't been cached for this type/params yet, create now + switch (primitiveCategory) { + case VARCHAR: + PrimitiveTypeEntry typeEntry = PrimitiveObjectInspectorUtils.getTypeEntryFromTypeSpecs( + primitiveCategory, + typeParams); + poi = new LazyHiveVarcharObjectInspector(typeEntry); + poi.setTypeParams(typeParams); + cachedParameterizedLazyObjectInspectors.setObjectInspector(poi); + break; + + default: + throw new RuntimeException( + "Primitve type " + primitiveCategory + " should not take parameters"); + } + } + + return poi; + } public static AbstractPrimitiveLazyObjectInspector getLazyObjectInspector( PrimitiveCategory primitiveCategory, boolean escaped, byte escapeChar) { - switch (primitiveCategory) { case BOOLEAN: return LAZY_BOOLEAN_OBJECT_INSPECTOR; @@ -99,6 +134,8 @@ public static LazyStringObjectInspector getLazyStringObjectInspector( return LAZY_DOUBLE_OBJECT_INSPECTOR; case STRING: return getLazyStringObjectInspector(escaped, escapeChar); + case VARCHAR: + return LAZY_VARCHAR_OBJECT_INSPECTOR; case BINARY: return LAZY_BINARY_OBJECT_INSPECTOR; case VOID: @@ -115,6 +152,24 @@ public static LazyStringObjectInspector getLazyStringObjectInspector( } } + public static AbstractPrimitiveLazyObjectInspector getLazyObjectInspector( + PrimitiveCategory primitiveCategory, BaseTypeParams typeParams, boolean escaped, byte escapeChar) { + + if (typeParams == null) { + return getLazyObjectInspector(primitiveCategory, escaped, escapeChar); + } else { + switch(primitiveCategory) { + case VARCHAR: + LazyHiveVarcharObjectInspector oi = (LazyHiveVarcharObjectInspector) + getParameterizedObjectInspector(primitiveCategory, typeParams); + return oi; + + default: + throw new RuntimeException("Type " + primitiveCategory + " does not take parameters"); + } + } + } + private LazyPrimitiveObjectInspectorFactory() { // prevent instantiation } diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java index 48b3c05..c3d31f1 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveVarcharObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBooleanObjectInspector; @@ -71,6 +72,8 @@ return new LazyBinaryDouble((WritableDoubleObjectInspector) oi); case STRING: return new LazyBinaryString((WritableStringObjectInspector) oi); + case VARCHAR: + return new LazyBinaryHiveVarchar((WritableHiveVarcharObjectInspector) oi); case VOID: // for NULL return new LazyBinaryVoid((WritableVoidObjectInspector) oi); case DATE: diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryHiveVarchar.java serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryHiveVarchar.java new file mode 100644 index 0000000..fb1b0ea --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryHiveVarchar.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.lazybinary; + +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; +import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveVarcharObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeParams; +import org.apache.hadoop.io.Text; + +public class LazyBinaryHiveVarchar extends + LazyBinaryPrimitive { + + protected int maxLength = -1; + + LazyBinaryHiveVarchar(WritableHiveVarcharObjectInspector oi) { + super(oi); + // Check for params + VarcharTypeParams typeParams = (VarcharTypeParams)oi.getTypeParams(); + if (typeParams == null) { + throw new RuntimeException("varchar type used without type params"); + } + maxLength = typeParams.length; + data = new HiveVarcharWritable(); + } + + LazyBinaryHiveVarchar(LazyBinaryHiveVarchar copy) { + super(copy); + maxLength = copy.maxLength; + data = new HiveVarcharWritable(copy.data); + } + + @Override + public void init(ByteArrayRef bytes, int start, int length) { + // re-use existing text member in varchar writable + Text textValue = data.getTextValue(); + textValue.set(bytes.getData(), start, length); + data.enforceMaxLength(maxLength); + } + +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java index 76acc6a..77a1951 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.serde2.lazybinary; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -44,6 +46,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; @@ -231,7 +235,7 @@ public Writable serialize(Object obj, ObjectInspector objInspector) * once already */ private static boolean serializeStruct(Output byteStream, Object obj, - StructObjectInspector soi, boolean warnedOnceNullMapKey) { + StructObjectInspector soi, boolean warnedOnceNullMapKey) throws SerDeException { // do nothing for null struct if (null == obj) { return warnedOnceNullMapKey; @@ -284,7 +288,8 @@ private static boolean serializeStruct(Output byteStream, Object obj, * once already */ public static boolean serialize(Output byteStream, Object obj, - ObjectInspector objInspector, boolean skipLengthPrefix, boolean warnedOnceNullMapKey) { + ObjectInspector objInspector, boolean skipLengthPrefix, boolean warnedOnceNullMapKey) + throws SerDeException { // do nothing for null object if (null == obj) { @@ -363,7 +368,24 @@ public static boolean serialize(Output byteStream, Object obj, byteStream.write(data, 0, length); return warnedOnceNullMapKey; } - + case VARCHAR: { + HiveVarcharObjectInspector hcoi = (HiveVarcharObjectInspector) poi; + String value = + hcoi.getPrimitiveWritableObject(obj).getHiveVarchar().getValue(); + int length = value.length(); + // Write byte size + if (!skipLengthPrefix) { + LazyBinaryUtils.writeVInt(byteStream, length); + } + // Write string value + try { + ByteBuffer bb = Text.encode(value); + byteStream.write(bb.array(), 0, bb.limit()); + } catch (CharacterCodingException err) { + throw new SerDeException(err); + } + return warnedOnceNullMapKey; + } case BINARY: { BinaryObjectInspector baoi = (BinaryObjectInspector) poi; BytesWritable bw = baoi.getPrimitiveWritableObject(obj); diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java index 6ad8704..69fd187 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.ParameterizedPrimitiveTypeUtils; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -196,6 +197,11 @@ public static void checkObjectByteInfo(ObjectInspector objectInspector, recordInfo.elementSize = vInt.value; break; + case VARCHAR: + LazyBinaryUtils.readVInt(bytes, offset, vInt); + recordInfo.elementOffset = vInt.length; + recordInfo.elementSize = vInt.value; + break; case BINARY: // using vint instead of 4 bytes LazyBinaryUtils.readVInt(bytes, offset, vInt); @@ -426,7 +432,8 @@ public static ObjectInspector getLazyBinaryObjectInspectorFromTypeInfo( case PRIMITIVE: { result = PrimitiveObjectInspectorFactory .getPrimitiveWritableObjectInspector(((PrimitiveTypeInfo) typeInfo) - .getPrimitiveCategory()); + .getPrimitiveCategory(), + ParameterizedPrimitiveTypeUtils.getTypeParamsFromTypeInfo(typeInfo)); break; } case LIST: { diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java index 5336643..0a8bb25 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaStringObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveVarcharObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBooleanObjectInspector; @@ -38,6 +39,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableTimestampObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.VoidObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableStringObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.ParameterizedPrimitiveTypeUtils; /** * ObjectInspectorConverters. @@ -101,6 +103,10 @@ private static Converter getConverter(PrimitiveObjectInspector inputOI, return new PrimitiveObjectInspectorConverter.StringConverter( inputOI); } + case VARCHAR: + return new PrimitiveObjectInspectorConverter.HiveVarcharConverter( + inputOI, + (SettableHiveVarcharObjectInspector) outputOI); case DATE: return new PrimitiveObjectInspectorConverter.DateConverter( inputOI, @@ -165,7 +171,8 @@ private static ObjectInspector getSettableConvertedOI( case PRIMITIVE: PrimitiveObjectInspector primInputOI = (PrimitiveObjectInspector) inputOI; return PrimitiveObjectInspectorFactory. - getPrimitiveWritableObjectInspector(primInputOI.getPrimitiveCategory()); + getPrimitiveWritableObjectInspector(primInputOI.getPrimitiveCategory(), + ParameterizedPrimitiveTypeUtils.getTypeParamsFromPrimitiveObjectInspector(primInputOI)); case STRUCT: return inputOI; case LIST: diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java index f7418bc..5340aa1 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java @@ -31,6 +31,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; @@ -42,6 +43,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; @@ -49,6 +51,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.BaseTypeParams; +import org.apache.hadoop.hive.serde2.typeinfo.ParameterizedPrimitiveTypeUtils; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.BytesWritable; @@ -86,8 +90,9 @@ public static ObjectInspector getWritableObjectInspector(ObjectInspector oi) { if (oi.getCategory() == Category.PRIMITIVE) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; if (!(poi instanceof AbstractPrimitiveWritableObjectInspector)) { - return PrimitiveObjectInspectorFactory - .getPrimitiveWritableObjectInspector(poi.getPrimitiveCategory()); + return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( + poi.getPrimitiveCategory(), + ParameterizedPrimitiveTypeUtils.getTypeParamsFromPrimitiveObjectInspector(poi)); } } return oi; @@ -108,25 +113,29 @@ public static ObjectInspector getStandardObjectInspector(ObjectInspector oi, switch (oi.getCategory()) { case PRIMITIVE: { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; + BaseTypeParams typeParams = + ParameterizedPrimitiveTypeUtils.getTypeParamsFromPrimitiveObjectInspector(poi); switch (objectInspectorOption) { case DEFAULT: { if (poi.preferWritable()) { - result = PrimitiveObjectInspectorFactory - .getPrimitiveWritableObjectInspector(poi.getPrimitiveCategory()); + result = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( + poi.getPrimitiveCategory(), + typeParams); } else { result = PrimitiveObjectInspectorFactory - .getPrimitiveJavaObjectInspector(poi.getPrimitiveCategory()); + .getPrimitiveJavaObjectInspector(poi.getPrimitiveCategory(), typeParams); } break; } case JAVA: { result = PrimitiveObjectInspectorFactory - .getPrimitiveJavaObjectInspector(poi.getPrimitiveCategory()); + .getPrimitiveJavaObjectInspector(poi.getPrimitiveCategory(), typeParams); break; } case WRITABLE: { - result = PrimitiveObjectInspectorFactory - .getPrimitiveWritableObjectInspector(poi.getPrimitiveCategory()); + result = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( + poi.getPrimitiveCategory(), + typeParams); break; } } @@ -487,6 +496,8 @@ public static int hashCode(Object o, ObjectInspector objIns) { } return r; } + case VARCHAR: + return ((HiveVarcharObjectInspector)poi).getPrimitiveWritableObject(o).hashCode(); case BINARY: return ((BinaryObjectInspector) poi).getPrimitiveWritableObject(o).hashCode(); @@ -682,6 +693,11 @@ public static int compare(Object o1, ObjectInspector oi1, Object o2, .compareTo(s2)); } } + case VARCHAR: { + HiveVarcharWritable t1 = ((HiveVarcharObjectInspector)poi1).getPrimitiveWritableObject(o1); + HiveVarcharWritable t2 = ((HiveVarcharObjectInspector)poi2).getPrimitiveWritableObject(o2); + return t1.compareTo(t2); + } case BINARY: { BytesWritable bw1 = ((BinaryObjectInspector) poi1).getPrimitiveWritableObject(o1); BytesWritable bw2 = ((BinaryObjectInspector) poi2).getPrimitiveWritableObject(o2); diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java index f1b8bdf..3e079fe 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hive.serde2.objectinspector; +import org.apache.hadoop.hive.serde2.typeinfo.BaseTypeParams; + /** * PrimitiveObjectInspector. * @@ -28,7 +30,7 @@ */ public static enum PrimitiveCategory { VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, - DATE, TIMESTAMP, BINARY, DECIMAL, UNKNOWN + DATE, TIMESTAMP, BINARY, DECIMAL, VARCHAR, UNKNOWN }; /** @@ -74,4 +76,17 @@ * most efficient way to getting data out of the Object. */ boolean preferWritable(); + + /** + * If the type has type parameters (such as varchar length, or decimal precision/scale), + * then return the parameters for the type. + * @return A BaseTypeParams object representing the parameters for the type, or null + */ + BaseTypeParams getTypeParams(); + + /** + * Set the type parameters for the type. + * @param newParams type parameters for the type + */ + void setTypeParams(BaseTypeParams newParams); } diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveObjectInspector.java index 8620af3..e42d7c6 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveObjectInspector.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveObjectInspector.java @@ -19,6 +19,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; +import org.apache.hadoop.hive.serde2.typeinfo.BaseTypeParams; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; /** * An AbstractPrimitiveObjectInspector is based on @@ -28,6 +30,7 @@ PrimitiveObjectInspector { transient PrimitiveTypeEntry typeEntry; + protected BaseTypeParams typeParams; /** * Construct a AbstractPrimitiveObjectInspector. @@ -76,7 +79,18 @@ public Category getCategory() { */ @Override public String getTypeName() { - return typeEntry.typeName; + return typeEntry.toString(); } + public BaseTypeParams getTypeParams() { + return typeParams; + } + + public void setTypeParams(BaseTypeParams newParams) { + if (typeParams != null && !typeEntry.isParameterized()) { + throw new UnsupportedOperationException( + "Attempting to add type parameters " + typeParams + " to type " + getTypeName()); + } + this.typeParams = newParams; + } } diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/HiveVarcharObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/HiveVarcharObjectInspector.java new file mode 100644 index 0000000..a3ae161 --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/HiveVarcharObjectInspector.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; + +public interface HiveVarcharObjectInspector extends PrimitiveObjectInspector { + HiveVarcharWritable getPrimitiveWritableObject(Object o); + + HiveVarchar getPrimitiveJavaObject(Object o); +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveVarcharObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveVarcharObjectInspector.java new file mode 100644 index 0000000..b072a3c --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveVarcharObjectInspector.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; +import org.apache.hadoop.hive.serde2.typeinfo.BaseTypeParams; +import org.apache.hadoop.hive.serde2.typeinfo.ParameterizedPrimitiveTypeUtils; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeParams; + +public class JavaHiveVarcharObjectInspector + extends AbstractPrimitiveJavaObjectInspector + implements SettableHiveVarcharObjectInspector { + + protected JavaHiveVarcharObjectInspector(PrimitiveTypeEntry typeEntry) { + super(typeEntry); + if (typeEntry.primitiveCategory != PrimitiveCategory.VARCHAR) { + throw new RuntimeException( + "TypeEntry of type varchar expected, got " + typeEntry.primitiveCategory); + } + } + + public HiveVarchar getPrimitiveJavaObject(Object o) { + if (o == null) { + return null; + } + HiveVarchar value = (HiveVarchar)o; + if (ParameterizedPrimitiveTypeUtils.doesPrimitiveMatchTypeParams( + value, (VarcharTypeParams) typeParams)) { + return value; + } + // value needs to be converted to match the type params (length, etc). + return getPrimitiveWithParams(value); + } + + @Override + public HiveVarcharWritable getPrimitiveWritableObject(Object o) { + if (o == null) { + return null; + } + return getWritableWithParams((HiveVarchar)o); + } + + private HiveVarchar getPrimitiveWithParams(HiveVarchar val) { + HiveVarchar hv = new HiveVarchar(val, getMaxLength()); + return hv; + } + + private HiveVarcharWritable getWritableWithParams(HiveVarchar val) { + HiveVarcharWritable newValue = new HiveVarcharWritable(); + newValue.set(val, getMaxLength()); + return newValue; + } + + @Override + public Object set(Object o, HiveVarchar value) { + HiveVarchar setValue = (HiveVarchar)o; + if (ParameterizedPrimitiveTypeUtils.doesPrimitiveMatchTypeParams( + value, (VarcharTypeParams) typeParams)) { + setValue.setValue(value); + } else { + // Otherwise value may be too long, convert to appropriate value based on params + setValue.setValue(value, getMaxLength()); + } + + return setValue; + } + + @Override + public Object set(Object o, String value) { + HiveVarchar convertedValue = (HiveVarchar)o; + convertedValue.setValue(value, getMaxLength()); + return convertedValue; + } + + @Override + public Object create(HiveVarchar value) { + HiveVarchar hc = new HiveVarchar(value, getMaxLength()); + return hc; + } + + public int getMaxLength() { + return typeParams != null ? ((VarcharTypeParams) typeParams).length : -1; + } +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java index 0f9df78..3b4acae 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java @@ -21,12 +21,15 @@ import java.sql.Date; import java.sql.Timestamp; +import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.serde2.ByteStream; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.lazy.LazyInteger; import org.apache.hadoop.hive.serde2.lazy.LazyLong; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeParams; import org.apache.hadoop.io.Text; /** @@ -394,6 +397,14 @@ public Text convert(Object input) { t.set(((StringObjectInspector) inputOI).getPrimitiveJavaObject(input)); } return t; + case VARCHAR: + // TODO: space padded value or stripped value when converting to string? + if (inputOI.preferWritable()) { + t.set(((HiveVarcharObjectInspector) inputOI).getPrimitiveWritableObject(input).toString()); + } else { + t.set(((HiveVarcharObjectInspector) inputOI).getPrimitiveJavaObject(input).toString()); + } + return t; case DATE: t.set(((DateObjectInspector) inputOI).getPrimitiveWritableObject(input).toString()); return t; @@ -430,4 +441,39 @@ public Object convert(Object input) { } } + + public static class HiveVarcharConverter implements Converter { + + PrimitiveObjectInspector inputOI; + SettableHiveVarcharObjectInspector outputOI; + HiveVarcharWritable hc; + + public HiveVarcharConverter(PrimitiveObjectInspector inputOI, SettableHiveVarcharObjectInspector outputOI) { + this.inputOI = inputOI; + this.outputOI = outputOI; + VarcharTypeParams typeParams = (VarcharTypeParams) outputOI.getTypeParams(); + + // unfortunately we seem to get instances of varchar object inspectors without params + // when an old-style UDF has an evaluate() method with varchar arguments. + // If we disallow varchar in old-style UDFs and only allow GenericUDFs to be defined + // with varchar arguments, then we might be able to enforce this properly. + //if (typeParams == null) { + // throw new RuntimeException("varchar type used without type params"); + //} + hc = new HiveVarcharWritable(); + } + + @Override + public Object convert(Object input) { + switch (inputOI.getPrimitiveCategory()) { + case BOOLEAN: + return outputOI.set(hc, + ((BooleanObjectInspector) inputOI).get(input) ? + new HiveVarchar("TRUE", -1) : new HiveVarchar("FALSE", -1)); + default: + return outputOI.set(hc, PrimitiveObjectInspectorUtils.getHiveVarchar(input, inputOI)); + } + } + + } } diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java index 281f730..6988003 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java @@ -20,6 +20,7 @@ import java.util.HashMap; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; @@ -29,7 +30,9 @@ import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.ParameterizedObjectInspectorMap; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; +import org.apache.hadoop.hive.serde2.typeinfo.BaseTypeParams; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.FloatWritable; @@ -65,6 +68,8 @@ new JavaDoubleObjectInspector(); public static final JavaStringObjectInspector javaStringObjectInspector = new JavaStringObjectInspector(); + public static final JavaHiveVarcharObjectInspector javaHiveVarcharObjectInspector = + new JavaHiveVarcharObjectInspector(PrimitiveObjectInspectorUtils.varcharTypeEntry); public static final JavaVoidObjectInspector javaVoidObjectInspector = new JavaVoidObjectInspector(); public static final JavaDateObjectInspector javaDateObjectInspector = @@ -92,6 +97,8 @@ new WritableDoubleObjectInspector(); public static final WritableStringObjectInspector writableStringObjectInspector = new WritableStringObjectInspector(); + public static final WritableHiveVarcharObjectInspector writableHiveVarcharObjectInspector = + new WritableHiveVarcharObjectInspector(PrimitiveObjectInspectorUtils.varcharTypeEntry); public static final WritableVoidObjectInspector writableVoidObjectInspector = new WritableVoidObjectInspector(); public static final WritableDateObjectInspector writableDateObjectInspector = @@ -122,6 +129,8 @@ writableDoubleObjectInspector); cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.STRING, writableStringObjectInspector); + cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.VARCHAR, + writableHiveVarcharObjectInspector); cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.VOID, writableVoidObjectInspector); cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.DATE, @@ -153,6 +162,8 @@ javaDoubleObjectInspector); cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.STRING, javaStringObjectInspector); + cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.VARCHAR, + javaHiveVarcharObjectInspector); cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.VOID, javaVoidObjectInspector); cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.DATE, @@ -166,6 +177,20 @@ } /** + * Cached Writable object inspectors for parameterized primitive types. + */ + private static ParameterizedObjectInspectorMap + cachedParameterizedPrimitiveWritableObjectInspectorCache = + new ParameterizedObjectInspectorMap(); + + /** + * Cached Java object inspectors for parameterized primitive types. + */ + private static ParameterizedObjectInspectorMap + cachedParameterizedPrimitiveJavaObjectInspectorCache = + new ParameterizedObjectInspectorMap(); + + /** * Returns the PrimitiveWritableObjectInspector for the PrimitiveCategory. * * @param primitiveCategory @@ -182,6 +207,53 @@ public static AbstractPrimitiveWritableObjectInspector getPrimitiveWritableObjec } /** + * Returns the PrimitiveWritableObjectInspector for the PrimitiveCategory, with option to + * pass in parameters for the primitive type (such as char(10)). + * Ideally this method should be used over the method without type parameters, + * and the type parameters (or lack of parameters) can be determined from + * the input ObjectInspector, TypeInfo, or TypeEntry. + * However there are situations where it is not possible to get any information about + * type parameters, such as when getting an object inspector based on reflection from + * the java or primitive class. + * @param primitiveCategory Primitve type category + * @param primitiveTypeParams Type parameters for the primitve type. + * Set to null if there are no type parameters + * @return + */ + public static AbstractPrimitiveWritableObjectInspector getPrimitiveWritableObjectInspector( + PrimitiveCategory primitiveCategory, + BaseTypeParams primitiveTypeParams) { + if (primitiveTypeParams == null) { + // No type params, just search the unparameterized types + return getPrimitiveWritableObjectInspector(primitiveCategory); + } else { + // Check our cached set of parameterized object inspectors for the primitive category, + // or create a new object inspector if one doesn't exist yet. + PrimitiveObjectInspector oi = + cachedParameterizedPrimitiveWritableObjectInspectorCache.getObjectInspector( + primitiveCategory, + primitiveTypeParams); + if (oi == null) { + // Do a bit of validation - not all primitive types use parameters. + switch (primitiveCategory) { + case VARCHAR: + PrimitiveTypeEntry typeEntry = PrimitiveObjectInspectorUtils.getTypeEntryFromTypeSpecs( + primitiveCategory, + primitiveTypeParams); + oi = new WritableHiveVarcharObjectInspector(typeEntry); + oi.setTypeParams(primitiveTypeParams); + cachedParameterizedPrimitiveWritableObjectInspectorCache.setObjectInspector(oi); + break; + default: + throw new RuntimeException( + "Primitve type " + primitiveCategory + " should not take parameters"); + } + } + return (AbstractPrimitiveWritableObjectInspector)oi; + } + } + + /** * Returns a PrimitiveWritableObjectInspector which implements ConstantObjectInspector * for the PrimitiveCategory. * @@ -207,6 +279,8 @@ public static ConstantObjectInspector getPrimitiveWritableConstantObjectInspecto return new WritableConstantDoubleObjectInspector((DoubleWritable)value); case STRING: return new WritableConstantStringObjectInspector((Text)value); + case VARCHAR: + return new WritableConstantHiveVarcharObjectInspector((HiveVarcharWritable)value); case DATE: return new WritableConstantDateObjectInspector((DateWritable)value); case TIMESTAMP: @@ -240,6 +314,52 @@ public static AbstractPrimitiveJavaObjectInspector getPrimitiveJavaObjectInspect } /** + * Returns the PrimitiveJavaObjectInspector for the PrimitiveCategory, with option to + * pass in parameters for the primitive type (such as char(10)). + * Ideally this method should be used over the method without type parameters, + * and the type parameters (or lack of parameters) can be determined from + * the input ObjectInspector, TypeInfo, or TypeEntry. + * However there are situations where it is not possible to get any information about + * type parameters, such as when getting an object inspector based on reflection from + * the java or primitive class. + * @param primitiveCategory Primitve type category + * @param primitiveTypeParams Type parameters for the primitve type. + * Set to null if there are no type parameters + * @return + */ + public static AbstractPrimitiveJavaObjectInspector getPrimitiveJavaObjectInspector( + PrimitiveCategory primitiveCategory, + BaseTypeParams primitiveTypeParams) { + if (primitiveTypeParams == null) { + // No type params, just search the unparameterized types + return getPrimitiveJavaObjectInspector(primitiveCategory); + } else { + // Check our cached set of parameterized object inspectors for the primitive category, + // or create a new object inspector if one doesn't exist yet. + PrimitiveObjectInspector oi = + cachedParameterizedPrimitiveJavaObjectInspectorCache.getObjectInspector( + primitiveCategory, + primitiveTypeParams); + if (oi == null) { + // Do a bit of validation - not all primitive types use parameters. + switch (primitiveCategory) { + case VARCHAR: + PrimitiveTypeEntry typeEntry = PrimitiveObjectInspectorUtils.getTypeEntryFromTypeSpecs( + primitiveCategory, + primitiveTypeParams); + oi = new JavaHiveVarcharObjectInspector(typeEntry); + oi.setTypeParams(primitiveTypeParams); + cachedParameterizedPrimitiveJavaObjectInspectorCache.setObjectInspector(oi); + break; + default: + throw new RuntimeException( + "Primitve type " + primitiveCategory + " should not take parameters"); + } + } + return (AbstractPrimitiveJavaObjectInspector)oi; + } + } + /** * Returns an ObjectInspector for a primitive Class. The Class can be a Hive * Writable class, or a Java Primitive Class. * diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java index 772eb43..5de681a 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java @@ -26,11 +26,16 @@ import java.util.HashMap; import java.util.Map; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; @@ -40,6 +45,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.typeinfo.BaseTypeParams; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeParams; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.FloatWritable; @@ -58,11 +65,12 @@ * ObjectInspector to return to the caller of SerDe2.getObjectInspector(). */ public final class PrimitiveObjectInspectorUtils { + private static Log LOG = LogFactory.getLog(PrimitiveObjectInspectorUtils.class); /** * TypeEntry stores information about a Hive Primitive TypeInfo. */ - public static class PrimitiveTypeEntry implements Writable { + public static class PrimitiveTypeEntry implements Writable, Cloneable { /** * The category of the PrimitiveType. @@ -87,15 +95,18 @@ * typeName is the name of the type as in DDL. */ public String typeName; + public Class typeParamsClass; + public BaseTypeParams typeParams; PrimitiveTypeEntry( PrimitiveObjectInspector.PrimitiveCategory primitiveCategory, String typeName, Class primitiveType, Class javaClass, - Class hiveClass) { + Class hiveClass, ClassparamsClass) { this.primitiveCategory = primitiveCategory; primitiveJavaType = primitiveType; primitiveJavaClass = javaClass; primitiveWritableClass = hiveClass; + typeParamsClass = paramsClass; this.typeName = typeName; } @@ -104,22 +115,105 @@ public void readFields(DataInput in) throws IOException { primitiveCategory = WritableUtils.readEnum(in, PrimitiveObjectInspector.PrimitiveCategory.class); typeName = WritableUtils.readString(in); + int typeParamsIndicator = WritableUtils.readVInt(in); try { primitiveJavaType = Class.forName(WritableUtils.readString(in)); primitiveJavaClass = Class.forName(WritableUtils.readString(in)); primitiveWritableClass = Class.forName(WritableUtils.readString(in)); + if (typeParamsIndicator == 1) { + typeParamsClass = Class.forName(WritableUtils.readString(in)); + typeParams = (BaseTypeParams)typeParamsClass.newInstance(); + typeParams.readFields(in); + } else { + typeParamsClass = null; + typeParams = null; + } } catch (ClassNotFoundException e) { throw new IOException(e); + } catch (IllegalAccessException e) { + throw new IOException(e); + } catch (InstantiationException e) { + throw new IOException(e); } } @Override public void write(DataOutput out) throws IOException { + int typeParamsIndicator = (isParameterized() && typeParams != null) ? 1 : 0; + WritableUtils.writeEnum(out, primitiveCategory); WritableUtils.writeString(out, typeName); + WritableUtils.writeVInt(out, typeParamsIndicator); WritableUtils.writeString(out, primitiveJavaType.getName()); WritableUtils.writeString(out, primitiveJavaClass.getName()); WritableUtils.writeString(out, primitiveWritableClass.getName()); + if (typeParamsIndicator == 1) { + WritableUtils.writeString(out, typeParamsClass.getName()); + typeParams.write(out); + } + } + + public PrimitiveTypeEntry addParameters(String[] parameters) { + if (parameters == null || parameters.length == 0) { + return this; + } + + PrimitiveTypeEntry result; + try { + BaseTypeParams newTypeParams = (BaseTypeParams)typeParamsClass.newInstance(); + newTypeParams.set(parameters); + String typeNameWithParams = this.typeName + newTypeParams.toString(); + if (typeNameToTypeEntry.containsKey(typeNameWithParams)) { + return typeNameToTypeEntry.get(typeNameWithParams); + } + result = (PrimitiveTypeEntry)this.clone(); + result.typeParams = newTypeParams; + + PrimitiveObjectInspectorUtils.addParameterizedType(result); + + return result; + } catch (Exception err) { + LOG.error("Error while setting type parameters: " + err); + return null; + } + } + + public boolean isParameterized() { + return (null != typeParamsClass); + } + + public Object clone() { + PrimitiveTypeEntry result = new PrimitiveTypeEntry( + this.primitiveCategory, + this.typeName, + this.primitiveJavaType, + this.primitiveJavaClass, + this.primitiveWritableClass, + this.typeParamsClass); + return result; + } + + public String toString() { + if (typeParams != null) { + return typeName + typeParams.toString(); + } + return typeName; + } + + public static BaseTypeParams createTypeParams(String typeName, String[] parameters) + throws SerDeException { + try { + PrimitiveTypeEntry typeEntry = getTypeEntryFromTypeName(typeName); + if (typeEntry != null && typeEntry.typeParamsClass != null) { + BaseTypeParams newTypeParams = (BaseTypeParams)typeEntry.typeParamsClass.newInstance(); + newTypeParams.set(parameters); + return newTypeParams; + } else { + return null; + } + } catch (Exception err) { + throw new SerDeException("Error creating type params for " + typeName, err); + } } } @@ -129,6 +223,10 @@ public void write(DataOutput out) throws IOException { static final Map, PrimitiveTypeEntry> primitiveWritableClassToTypeEntry = new HashMap, PrimitiveTypeEntry>(); static final Map typeNameToTypeEntry = new HashMap(); + static void addParameterizedType(PrimitiveTypeEntry t) { + typeNameToTypeEntry.put(t.toString(), t); + } + static void registerType(PrimitiveTypeEntry t) { if (t.primitiveCategory != PrimitiveCategory.UNKNOWN) { primitiveCategoryToTypeEntry.put(t.primitiveCategory, t); @@ -149,53 +247,57 @@ static void registerType(PrimitiveTypeEntry t) { public static final PrimitiveTypeEntry binaryTypeEntry = new PrimitiveTypeEntry( PrimitiveCategory.BINARY, serdeConstants.BINARY_TYPE_NAME, byte[].class, - byte[].class, BytesWritable.class); + byte[].class, BytesWritable.class, null); public static final PrimitiveTypeEntry stringTypeEntry = new PrimitiveTypeEntry( PrimitiveCategory.STRING, serdeConstants.STRING_TYPE_NAME, null, String.class, - Text.class); + Text.class, null); public static final PrimitiveTypeEntry booleanTypeEntry = new PrimitiveTypeEntry( PrimitiveCategory.BOOLEAN, serdeConstants.BOOLEAN_TYPE_NAME, Boolean.TYPE, - Boolean.class, BooleanWritable.class); + Boolean.class, BooleanWritable.class, null); public static final PrimitiveTypeEntry intTypeEntry = new PrimitiveTypeEntry( PrimitiveCategory.INT, serdeConstants.INT_TYPE_NAME, Integer.TYPE, - Integer.class, IntWritable.class); + Integer.class, IntWritable.class, null); public static final PrimitiveTypeEntry longTypeEntry = new PrimitiveTypeEntry( PrimitiveCategory.LONG, serdeConstants.BIGINT_TYPE_NAME, Long.TYPE, - Long.class, LongWritable.class); + Long.class, LongWritable.class, null); public static final PrimitiveTypeEntry floatTypeEntry = new PrimitiveTypeEntry( PrimitiveCategory.FLOAT, serdeConstants.FLOAT_TYPE_NAME, Float.TYPE, - Float.class, FloatWritable.class); + Float.class, FloatWritable.class, null); public static final PrimitiveTypeEntry voidTypeEntry = new PrimitiveTypeEntry( PrimitiveCategory.VOID, serdeConstants.VOID_TYPE_NAME, Void.TYPE, Void.class, - NullWritable.class); + NullWritable.class, null); // No corresponding Writable classes for the following 3 in hadoop 0.17.0 public static final PrimitiveTypeEntry doubleTypeEntry = new PrimitiveTypeEntry( PrimitiveCategory.DOUBLE, serdeConstants.DOUBLE_TYPE_NAME, Double.TYPE, - Double.class, DoubleWritable.class); + Double.class, DoubleWritable.class, null); public static final PrimitiveTypeEntry byteTypeEntry = new PrimitiveTypeEntry( PrimitiveCategory.BYTE, serdeConstants.TINYINT_TYPE_NAME, Byte.TYPE, - Byte.class, ByteWritable.class); + Byte.class, ByteWritable.class, null); public static final PrimitiveTypeEntry shortTypeEntry = new PrimitiveTypeEntry( PrimitiveCategory.SHORT, serdeConstants.SMALLINT_TYPE_NAME, Short.TYPE, - Short.class, ShortWritable.class); + Short.class, ShortWritable.class, null); public static final PrimitiveTypeEntry dateTypeEntry = new PrimitiveTypeEntry( PrimitiveCategory.DATE, serdeConstants.DATE_TYPE_NAME, null, - Date.class, DateWritable.class); + Date.class, DateWritable.class, null); public static final PrimitiveTypeEntry timestampTypeEntry = new PrimitiveTypeEntry( PrimitiveCategory.TIMESTAMP, serdeConstants.TIMESTAMP_TYPE_NAME, null, - Timestamp.class, TimestampWritable.class); + Timestamp.class, TimestampWritable.class, null); public static final PrimitiveTypeEntry decimalTypeEntry = new PrimitiveTypeEntry( PrimitiveCategory.DECIMAL, serdeConstants.DECIMAL_TYPE_NAME, null, - HiveDecimal.class, HiveDecimalWritable.class); + HiveDecimal.class, HiveDecimalWritable.class, null); + public static final PrimitiveTypeEntry varcharTypeEntry = new PrimitiveTypeEntry( + PrimitiveCategory.VARCHAR, serdeConstants.VARCHAR_TYPE_NAME, null, HiveVarchar.class, + HiveVarcharWritable.class, VarcharTypeParams.class); // The following is a complex type for special handling public static final PrimitiveTypeEntry unknownTypeEntry = new PrimitiveTypeEntry( - PrimitiveCategory.UNKNOWN, "unknown", null, Object.class, null); + PrimitiveCategory.UNKNOWN, "unknown", null, Object.class, null, null); static { registerType(binaryTypeEntry); registerType(stringTypeEntry); + registerType(varcharTypeEntry); registerType(booleanTypeEntry); registerType(intTypeEntry); registerType(longTypeEntry); @@ -317,6 +419,23 @@ public static PrimitiveTypeEntry getTypeEntryFromTypeName(String typeName) { return typeNameToTypeEntry.get(typeName); } + public static PrimitiveTypeEntry getTypeEntryFromTypeSpecs( + PrimitiveCategory primitiveCategory, + BaseTypeParams typeParams) { + String typeString = primitiveCategory.toString().toLowerCase(); + if (typeParams != null) { + typeString += typeParams.toString(); + } + PrimitiveTypeEntry typeEntry = getTypeEntryFromTypeName(typeString); + if (typeEntry == null) { + // Parameterized type doesn't exist yet, create now. + typeEntry = (PrimitiveTypeEntry)getTypeEntryFromTypeSpecs(primitiveCategory, null).clone(); + typeEntry.typeParams = typeParams; + addParameterizedType(typeEntry); + } + return typeEntry; + } + /** * Compare 2 primitive objects. Conversion not allowed. Note that NULL does * not equal to NULL according to SQL standard. @@ -366,6 +485,10 @@ public static boolean comparePrimitiveObjects(Object o1, .getPrimitiveWritableObject(o2); return t1.equals(t2); } + case VARCHAR: { + return ((HiveVarcharObjectInspector)oi1).getPrimitiveWritableObject(o1) + .equals(((HiveVarcharObjectInspector)oi2).getPrimitiveWritableObject(o2)); + } case DATE: { return ((DateObjectInspector) oi1).getPrimitiveWritableObject(o1) .equals(((DateObjectInspector) oi2).getPrimitiveWritableObject(o2)); @@ -569,6 +692,10 @@ public static int getInt(Object o, PrimitiveObjectInspector oi) { } break; } + case VARCHAR: { + result = Integer.parseInt(getString(o, oi)); + break; + } case TIMESTAMP: result = (int) (((TimestampObjectInspector) oi) .getPrimitiveWritableObject(o).getSeconds()); @@ -628,6 +755,10 @@ public static long getLong(Object o, PrimitiveObjectInspector oi) { result = Long.parseLong(s); } break; + case VARCHAR: { + result = Long.parseLong(getString(o, oi)); + break; + } case TIMESTAMP: result = ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o) .getSeconds(); @@ -681,6 +812,9 @@ public static double getDouble(Object o, PrimitiveObjectInspector oi) { String s = soi.getPrimitiveJavaObject(o); result = Double.parseDouble(s); break; + case VARCHAR: + result = Double.parseDouble(getString(o, oi)); + break; case TIMESTAMP: result = ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o).getDouble(); break; @@ -746,6 +880,10 @@ public static String getString(Object o, PrimitiveObjectInspector oi) { StringObjectInspector soi = (StringObjectInspector) oi; result = soi.getPrimitiveJavaObject(o); break; + case VARCHAR: + HiveVarcharObjectInspector hcoi = (HiveVarcharObjectInspector) oi; + result = hcoi.getPrimitiveJavaObject(o).toString(); + break; case DATE: result = ((DateObjectInspector) oi).getPrimitiveWritableObject(o).toString(); break; @@ -763,6 +901,28 @@ public static String getString(Object o, PrimitiveObjectInspector oi) { return result; } + public static HiveVarchar getHiveVarchar(Object o, PrimitiveObjectInspector oi) { + + if (o == null) { + return null; + } + + HiveVarchar result = null; + switch (oi.getPrimitiveCategory()) { + case VARCHAR: + result = ((HiveVarcharObjectInspector)oi).getPrimitiveJavaObject(o); + break; + default: + // Is there a way to provide char length here? + // It might actually be ok as long as there is an object inspector (with char length) + // receiving this value. + result = new HiveVarchar(); + result.setValue(getString(o, oi)); + break; + } + return result; + } + public static BytesWritable getBinary(Object o, PrimitiveObjectInspector oi) { if (null == o) { @@ -826,6 +986,9 @@ public static HiveDecimal getHiveDecimal(Object o, PrimitiveObjectInspector oi) case STRING: result = new HiveDecimal(((StringObjectInspector) oi).getPrimitiveJavaObject(o)); break; + case VARCHAR: + result = new HiveDecimal(getString(o, oi)); + break; case TIMESTAMP: Double ts = ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o) .getDouble(); @@ -861,6 +1024,15 @@ public static Date getDate(Object o, PrimitiveObjectInspector oi) { result = null; } break; + case VARCHAR: { + try { + String val = getString(o, oi).trim(); + result = Date.valueOf(val); + } catch (IllegalArgumentException e) { + result = null; + } + break; + } case DATE: result = ((DateObjectInspector) oi).getPrimitiveWritableObject(o).get(); break; @@ -913,20 +1085,11 @@ public static Timestamp getTimestamp(Object o, PrimitiveObjectInspector oi) { break; case STRING: StringObjectInspector soi = (StringObjectInspector) oi; - String s = soi.getPrimitiveJavaObject(o).trim(); - - // Throw away extra if more than 9 decimal places - int periodIdx = s.indexOf("."); - if (periodIdx != -1) { - if (s.length() - periodIdx > 9) { - s = s.substring(0, periodIdx + 10); - } - } - try { - result = Timestamp.valueOf(s); - } catch (IllegalArgumentException e) { - result = null; - } + String s = soi.getPrimitiveJavaObject(o); + result = getTimestampFromString(s); + break; + case VARCHAR: + result = getTimestampFromString(getString(o, oi)); break; case DATE: result = new Timestamp( @@ -942,6 +1105,25 @@ public static Timestamp getTimestamp(Object o, PrimitiveObjectInspector oi) { return result; } + static Timestamp getTimestampFromString(String s) { + Timestamp result; + s = s.trim(); + + // Throw away extra if more than 9 decimal places + int periodIdx = s.indexOf("."); + if (periodIdx != -1) { + if (s.length() - periodIdx > 9) { + s = s.substring(0, periodIdx + 10); + } + } + try { + result = Timestamp.valueOf(s); + } catch (IllegalArgumentException e) { + result = null; + } + return result; + } + public static Class getJavaPrimitiveClassFromObjectInspector(ObjectInspector oi) { if (oi.getCategory() != Category.PRIMITIVE) { return null; @@ -952,8 +1134,72 @@ public static Timestamp getTimestamp(Object o, PrimitiveObjectInspector oi) { return t == null ? null : t.primitiveJavaClass; } + /** + * Provide a general grouping for each primitive data type. + */ + public static enum PrimitiveGrouping { + NUMERIC_GROUP, STRING_GROUP, BOOLEAN_GROUP, DATE_GROUP, BINARY_GROUP, UNKNOWN_GROUP + }; + + public static PrimitiveGrouping getPrimitiveGrouping(PrimitiveCategory primitiveCategory) { + switch (primitiveCategory) { + case BYTE: + case SHORT: + case INT: + case LONG: + case FLOAT: + case DOUBLE: + case DECIMAL: + return PrimitiveGrouping.NUMERIC_GROUP; + case STRING: + case VARCHAR: + return PrimitiveGrouping.STRING_GROUP; + case BOOLEAN: + return PrimitiveGrouping.BOOLEAN_GROUP; + case TIMESTAMP: + case DATE: + return PrimitiveGrouping.DATE_GROUP; + case BINARY: + return PrimitiveGrouping.BINARY_GROUP; + default: + return PrimitiveGrouping.UNKNOWN_GROUP; + } + } + private PrimitiveObjectInspectorUtils() { // prevent instantiation } + /** + * Helper class to store parameterized primitive object inspectors, which can be + * used by the various object inspector factory methods. + */ + public static class ParameterizedObjectInspectorMap { + HashMap> entries; + + public ParameterizedObjectInspectorMap() { + entries = + new HashMap>(); + } + + public PrimitiveObjectInspector getObjectInspector( + PrimitiveCategory category, + BaseTypeParams params) { + HashMap entriesForCategory = entries.get(category); + if (entriesForCategory == null) { + return null; + } + return (PrimitiveObjectInspector)entriesForCategory.get(params.toString()); + } + + public void setObjectInspector(PrimitiveObjectInspector oi) { + PrimitiveCategory category = oi.getPrimitiveCategory(); + HashMap entriesForCategory = entries.get(category); + if (entriesForCategory == null) { + entriesForCategory = new HashMap(); + entries.put(category, entriesForCategory); + } + entriesForCategory.put(oi.getTypeParams().toString(), oi); + } + } } diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableHiveVarcharObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableHiveVarcharObjectInspector.java new file mode 100644 index 0000000..4f75f13 --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableHiveVarcharObjectInspector.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; + +public interface SettableHiveVarcharObjectInspector extends HiveVarcharObjectInspector { + Object set(Object o, HiveVarchar value); + + Object set(Object o, String value); + + Object create(HiveVarchar value); + +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantHiveVarcharObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantHiveVarcharObjectInspector.java new file mode 100644 index 0000000..e70259e --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantHiveVarcharObjectInspector.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeParams; + +/** + * A WritableConstantHiveVarcharObjectInspector is a WritableHiveVarcharObjectInspector + * that implements ConstantObjectInspector. + */ +public class WritableConstantHiveVarcharObjectInspector extends + WritableHiveVarcharObjectInspector implements + ConstantObjectInspector { + + protected HiveVarcharWritable value; + + WritableConstantHiveVarcharObjectInspector(HiveVarcharWritable value) { + super(PrimitiveObjectInspectorUtils.varcharTypeEntry); + this.value = value; + + // Determine character length and update type params/typeinfo accordingly. + VarcharTypeParams typeParams = new VarcharTypeParams(); + typeParams.length = this.value.getCharacterLength(); + setTypeParams(typeParams); + this.typeEntry = PrimitiveObjectInspectorUtils.getTypeEntryFromTypeSpecs( + PrimitiveCategory.VARCHAR, + typeParams); + } + + @Override + public HiveVarcharWritable getWritableConstantValue() { + return value; + } +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java new file mode 100644 index 0000000..cbfeae8 --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; +import org.apache.hadoop.hive.serde2.typeinfo.BaseTypeParams; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeParams; +import org.apache.hadoop.hive.serde2.typeinfo.ParameterizedPrimitiveTypeUtils; + +public class WritableHiveVarcharObjectInspector + extends AbstractPrimitiveWritableObjectInspector + implements SettableHiveVarcharObjectInspector { + + private static final Log LOG = LogFactory.getLog(WritableHiveVarcharObjectInspector.class); + + protected WritableHiveVarcharObjectInspector(PrimitiveTypeEntry typeEntry) { + super(typeEntry); + if (typeEntry.primitiveCategory != PrimitiveCategory.VARCHAR) { + throw new RuntimeException( + "TypeEntry of type varchar expected, got " + typeEntry.primitiveCategory); + } + } + + @Override + public HiveVarchar getPrimitiveJavaObject(Object o) { + // check input object's length, if it doesn't match + // then output a new primitive with the correct params. + if (o == null) { + return null; + } + HiveVarcharWritable writable = ((HiveVarcharWritable)o); + if (doesWritableMatchTypeParams(writable)) { + return writable.getHiveVarchar(); + } + return getPrimitiveWithParams(writable); + } + + public HiveVarcharWritable getPrimitiveWritableObject(Object o) { + // check input object's length, if it doesn't match + // then output new writable with correct params. + if (o == null) { + return null; + } + HiveVarcharWritable writable = ((HiveVarcharWritable)o); + if (doesWritableMatchTypeParams((HiveVarcharWritable)o)) { + return writable; + } + + return getWritableWithParams(writable); + } + + private HiveVarchar getPrimitiveWithParams(HiveVarcharWritable val) { + HiveVarchar hv = new HiveVarchar(); + hv.setValue(val.getHiveVarchar(), getMaxLength()); + return hv; + } + + private HiveVarcharWritable getWritableWithParams(HiveVarcharWritable val) { + HiveVarcharWritable newValue = new HiveVarcharWritable(); + newValue.set(val, getMaxLength()); + return newValue; + } + + private boolean doesWritableMatchTypeParams(HiveVarcharWritable writable) { + return ParameterizedPrimitiveTypeUtils.doesWritableMatchTypeParams( + writable, (VarcharTypeParams) typeParams); + } + + private boolean doesPrimitiveMatchTypeParams(HiveVarchar value) { + return ParameterizedPrimitiveTypeUtils.doesPrimitiveMatchTypeParams( + value, (VarcharTypeParams) typeParams); + } + + @Override + public Object copyObject(Object o) { + if (o == null) { + return null; + } + HiveVarcharWritable writable = (HiveVarcharWritable)o; + if (doesWritableMatchTypeParams((HiveVarcharWritable)o)) { + return new HiveVarcharWritable(writable); + } + return getWritableWithParams(writable); + } + + @Override + public Object set(Object o, HiveVarchar value) { + HiveVarcharWritable writable = (HiveVarcharWritable)o; + writable.set(value, getMaxLength()); + return o; + } + + @Override + public Object set(Object o, String value) { + HiveVarcharWritable writable = (HiveVarcharWritable)o; + writable.set(value, getMaxLength()); + return o; + } + + @Override + public Object create(HiveVarchar value) { + HiveVarcharWritable ret; + ret = new HiveVarcharWritable(); + ret.set(value, getMaxLength()); + return ret; + } + + public int getMaxLength() { + return typeParams != null ? ((VarcharTypeParams) typeParams).length : -1; + } +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseTypeParams.java serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseTypeParams.java new file mode 100644 index 0000000..1eba561 --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseTypeParams.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.typeinfo; + +import java.io.Serializable; + +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.io.Writable; + +/** + * Base type for type-specific params, such as char(10) or decimal(10, 2). + */ +public abstract class BaseTypeParams implements Writable, Serializable { + + private static final long serialVersionUID = 1L; + + public abstract void validateParams() throws SerDeException; + + public abstract void populateParams(String[] params) throws SerDeException; + + public abstract String toString(); + + public void set(String[] params) throws SerDeException { + populateParams(params); + validateParams(); + } + + // Needed for conversion to/from TypeQualifiers. Override in subclasses. + public boolean hasCharacterMaximumLength() { + return false; + } + public int getCharacterMaximumLength() { + return -1; + } +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/ParameterizedPrimitiveTypeUtils.java serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/ParameterizedPrimitiveTypeUtils.java new file mode 100644 index 0000000..3c3fde2 --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/ParameterizedPrimitiveTypeUtils.java @@ -0,0 +1,69 @@ +package org.apache.hadoop.hive.serde2.typeinfo; + +import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +public class ParameterizedPrimitiveTypeUtils { + + public static BaseTypeParams getTypeParamsFromTypeInfo(TypeInfo typeInfo) { + BaseTypeParams typeParams = null; + if (typeInfo instanceof PrimitiveTypeInfo) { + PrimitiveTypeInfo ppti = (PrimitiveTypeInfo)typeInfo; + typeParams = ppti.getParameters(); + } + return typeParams; + } + + public static BaseTypeParams getTypeParamsFromPrimitiveTypeEntry(PrimitiveTypeEntry typeEntry) { + return typeEntry.typeParams; + } + + public static BaseTypeParams getTypeParamsFromPrimitiveObjectInspector( + PrimitiveObjectInspector oi) { + return oi.getTypeParams(); + } + + /** + * Utils for varchar type + */ + public static class HiveVarcharSerDeHelper { + public int maxLength; + public HiveVarcharWritable writable = new HiveVarcharWritable(); + + public HiveVarcharSerDeHelper(VarcharTypeParams typeParams) { + if (typeParams == null) { + throw new RuntimeException("varchar type used without type params"); + } + maxLength = typeParams.getLength(); + } + } + + public static boolean doesWritableMatchTypeParams(HiveVarcharWritable writable, VarcharTypeParams typeParams) { + return (typeParams == null || typeParams.length >= writable.getCharacterLength()); + } + + public static boolean doesPrimitiveMatchTypeParams(HiveVarchar value, VarcharTypeParams typeParams) { + return (typeParams == null || typeParams.length == value.getCharacterLength()); + } +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java index 46d3f3d..fc87a43 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; /** * There are limited number of Primitive Types. All Primitive Types are defined @@ -31,11 +32,12 @@ * Always use the TypeInfoFactory to create new TypeInfo objects, instead of * directly creating an instance of this class. */ -public final class PrimitiveTypeInfo extends TypeInfo implements Serializable { +public class PrimitiveTypeInfo extends TypeInfo implements Serializable { private static final long serialVersionUID = 1L; - private String typeName; + protected String typeName; + protected BaseTypeParams typeParams; /** * For java serialization use only. @@ -59,7 +61,7 @@ public Category getCategory() { } public PrimitiveCategory getPrimitiveCategory() { - return PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(typeName).primitiveCategory; + return getPrimitiveTypeEntry().primitiveCategory; } public Class getPrimitiveWritableClass() { @@ -81,6 +83,36 @@ public String getTypeName() { } /** + * If the type has type parameters (such as varchar length, or decimal precision/scale), + * then return the parameters for the type. + * @return A BaseTypeParams object representing the parameters for the type, or null + */ + public BaseTypeParams getParameters() { + return typeParams; + } + + /** + * Set the type parameters for the type. + * @param typeParams type parameters for the type + */ + public void setParameters(BaseTypeParams typeParams) { + // Ideally could check here to make sure the type really supports parameters, + // however during deserialization some of the required fields are not set at the + // time that the type params are set. We would have to customize the way this class + // is serialized/deserialized for the check to work. + //if (typeParams != null && !getPrimitiveTypeEntry().isParameterized()) { + // throw new UnsupportedOperationException( + // "Attempting to add type parameters " + typeParams + " to type " + getTypeName()); + //} + this.typeParams = typeParams; + } + + public PrimitiveTypeEntry getPrimitiveTypeEntry() { + return PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName( + TypeInfoUtils.getBaseName(typeName)); + } + + /** * Compare if 2 TypeInfos are the same. We use TypeInfoFactory to cache * TypeInfos, so we only need to compare the Object pointer. */ @@ -97,4 +129,7 @@ public int hashCode() { return typeName.hashCode(); } + public String toString() { + return typeName; + } } diff --git serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java index 55c1069..36a7008 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java @@ -49,6 +49,15 @@ protected TypeInfo() { */ public abstract String getTypeName(); + /** + * String representing the qualified type name. + * Qualified types should override this method. + * @return + */ + public String getQualifiedName() { + return getTypeName(); + } + @Override public String toString() { return getTypeName(); diff --git serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java index 542af09..cb7e330 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java @@ -22,8 +22,12 @@ import java.util.List; import java.util.concurrent.ConcurrentHashMap; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; /** * TypeInfoFactory can be used to create the TypeInfo object for any types. @@ -33,7 +37,7 @@ * objects that represents the same type. */ public final class TypeInfoFactory { - + private static Log LOG = LogFactory.getLog(TypeInfoFactory.class); static ConcurrentHashMap cachedPrimitiveTypeInfo = new ConcurrentHashMap(); private TypeInfoFactory() { @@ -41,13 +45,40 @@ private TypeInfoFactory() { } public static TypeInfo getPrimitiveTypeInfo(String typeName) { - if (null == PrimitiveObjectInspectorUtils - .getTypeEntryFromTypeName(typeName)) { + PrimitiveTypeEntry typeEntry = PrimitiveObjectInspectorUtils + .getTypeEntryFromTypeName(TypeInfoUtils.getBaseName(typeName)); + if (null == typeEntry) { throw new RuntimeException("Cannot getPrimitiveTypeInfo for " + typeName); } TypeInfo result = cachedPrimitiveTypeInfo.get(typeName); if (result == null) { - result = new PrimitiveTypeInfo(typeName); + TypeInfoUtils.PrimitiveParts parts = TypeInfoUtils.parsePrimitiveParts(typeName); + // Create params if there are any + if (parts.typeParams != null && parts.typeParams.length > 0) { + // The type string came with parameters. Parse and add to TypeInfo + try { + BaseTypeParams typeParams = PrimitiveTypeEntry.createTypeParams(parts.typeName, parts.typeParams); + result = new PrimitiveTypeInfo(typeName); + ((PrimitiveTypeInfo) result).setParameters(typeParams); + } catch (Exception err) { + LOG.error(err); + result = null; + } + } else { + // No type params + + // Prevent creation of varchar TypeInfo with no length specification. + // This can happen if an old-style UDF uses a varchar type either as an + // argument or return type in an evaluate() function, or other instances + // of using reflection-based methods for retrieving a TypeInfo. + if (typeEntry.primitiveCategory == PrimitiveCategory.VARCHAR) { + LOG.error("varchar type used with no type params"); + throw new RuntimeException("varchar type used with no type params"); + } + + result = new PrimitiveTypeInfo(parts.typeName); + } + cachedPrimitiveTypeInfo.put(typeName, result); } return result; @@ -66,6 +97,8 @@ public static TypeInfo getPrimitiveTypeInfo(String typeName) { public static final TypeInfo timestampTypeInfo = getPrimitiveTypeInfo(serdeConstants.TIMESTAMP_TYPE_NAME); public static final TypeInfo binaryTypeInfo = getPrimitiveTypeInfo(serdeConstants.BINARY_TYPE_NAME); public static final TypeInfo decimalTypeInfo = getPrimitiveTypeInfo(serdeConstants.DECIMAL_TYPE_NAME); + // Disallow usage of varchar without length specifier. + //public static final TypeInfo varcharTypeInfo = getPrimitiveTypeInfo(serdeConstants.VARCHAR_TYPE_NAME); public static final TypeInfo unknownTypeInfo = getPrimitiveTypeInfo("unknown"); diff --git serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java index 84e0b2e..88d1d36 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java @@ -22,23 +22,25 @@ import java.lang.reflect.Method; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; -import java.util.concurrent.ConcurrentHashMap; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; @@ -56,7 +58,7 @@ private TypeInfoUtils() { /** * Return the extended TypeInfo from a Java type. By extended TypeInfo, we * allow unknownType for java.lang.Object. - * + * * @param t * The Java type. * @param m @@ -148,7 +150,7 @@ public static Type getArrayElementType(Type t) { /** * Get the parameter TypeInfo for a method. - * + * * @param size * In case the last parameter of Method is an array, we will try to * return a List with the specified size by repeating the @@ -194,12 +196,46 @@ public static Type getArrayElementType(Type t) { return typeInfos; } + public static boolean hasParameters(String typeName) { + int idx = typeName.indexOf('('); + if (idx == -1) { + return false; + } else { + return true; + } + } + + public static String getBaseName(String typeName) { + int idx = typeName.indexOf('('); + if (idx == -1) { + return typeName; + } else { + return typeName.substring(0, idx); + } + } + + /** + * returns true if both TypeInfos are of primitive type, and the primitive category matches. + * @param ti1 + * @param ti2 + * @return + */ + public static boolean doPrimitiveCategoriesMatch(TypeInfo ti1, TypeInfo ti2) { + if (ti1.getCategory() == Category.PRIMITIVE && ti2.getCategory() == Category.PRIMITIVE) { + if (((PrimitiveTypeInfo)ti1).getPrimitiveCategory() + == ((PrimitiveTypeInfo)ti2).getPrimitiveCategory()) { + return true; + } + } + return false; + } + /** * Parse a recursive TypeInfo list String. For example, the following inputs * are valid inputs: * "int,string,map,list>>,list>" * The separators between TypeInfos can be ",", ":", or ";". - * + * * In order to use this class: TypeInfoParser parser = new * TypeInfoParser("int,string"); ArrayList typeInfos = * parser.parseTypeInfos(); @@ -225,7 +261,7 @@ private static boolean isTypeChar(char c) { * Tokenize the typeInfoString. The rule is simple: all consecutive * alphadigits and '_', '.' are in one token, and all other characters are * one character per token. - * + * * tokenize("map") should return * ["map","<","int",",","string",">"] */ @@ -281,6 +317,14 @@ public TypeInfoParser(String typeInfoString) { return typeInfos; } + private Token peek() { + if (iToken < typeInfoTokens.size()) { + return typeInfoTokens.get(iToken); + } else { + return null; + } + } + private Token expect(String item) { return expect(item, null); } @@ -320,6 +364,27 @@ private Token expect(String item, String alternative) { return t; } + private String[] parseParams() { + List params = new LinkedList(); + + Token t = peek(); + if (t != null && t.text.equals("(")) { + expect("("); + + // checking for null in the for-loop condition prevents null-ptr exception + // and allows us to fail more gracefully with a parsing error. + for(t = peek(); (t == null) || !t.text.equals(")"); t = expect(",",")")) { + params.add(expect("name").text); + } + if (params.size() == 0) { + throw new IllegalArgumentException( + "type parameters expected for type string " + typeInfoString); + } + } + + return params.toArray(new String[params.size()]); + } + private TypeInfo parseType() { Token t = expect("type"); @@ -329,7 +394,11 @@ private TypeInfo parseType() { .getTypeEntryFromTypeName(t.text); if (primitiveType != null && !primitiveType.primitiveCategory.equals(PrimitiveCategory.UNKNOWN)) { - return TypeInfoFactory.getPrimitiveTypeInfo(primitiveType.typeName); + if (primitiveType.isParameterized()) { + primitiveType = primitiveType.addParameters(parseParams()); + } + // If type has qualifiers, the TypeInfo needs them in its type string + return TypeInfoFactory.getPrimitiveTypeInfo(primitiveType.toString()); } // Is this a list type? @@ -399,6 +468,26 @@ private TypeInfo parseType() { + t.position + " of '" + typeInfoString + "'"); } + public PrimitiveParts parsePrimitiveParts() { + PrimitiveParts parts = new PrimitiveParts(); + Token t = expect("type"); + parts.typeName = t.text; + parts.typeParams = parseParams(); + return parts; + } + } + + public static class PrimitiveParts { + public String typeName; + public String[] typeParams; + } + + /** + * Make some of the TypeInfo parsing available as a utility. + */ + public static PrimitiveParts parsePrimitiveParts(String typeInfoString) { + TypeInfoParser parser = new TypeInfoParser(typeInfoString); + return parser.parsePrimitiveParts(); } static Map cachedStandardObjectInspector = @@ -414,9 +503,9 @@ public static ObjectInspector getStandardWritableObjectInspectorFromTypeInfo( if (result == null) { switch (typeInfo.getCategory()) { case PRIMITIVE: { - result = PrimitiveObjectInspectorFactory - .getPrimitiveWritableObjectInspector(((PrimitiveTypeInfo) typeInfo) - .getPrimitiveCategory()); + result = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( + ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory(), + ParameterizedPrimitiveTypeUtils.getTypeParamsFromTypeInfo(typeInfo)); break; } case LIST: { @@ -495,7 +584,9 @@ public static ObjectInspector getStandardJavaObjectInspectorFromTypeInfo( // StandardPrimitiveObjectInspector result = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector(PrimitiveObjectInspectorUtils - .getTypeEntryFromTypeName(typeInfo.getTypeName()).primitiveCategory); + .getTypeEntryFromTypeName( + typeInfo.getTypeName()).primitiveCategory, + ParameterizedPrimitiveTypeUtils.getTypeParamsFromTypeInfo(typeInfo)); break; } case LIST: { diff --git serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeParams.java serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeParams.java new file mode 100644 index 0000000..5f30243 --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeParams.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.typeinfo; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.io.Serializable; + +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.io.WritableUtils; + +public class VarcharTypeParams extends BaseTypeParams implements Serializable { + private static final long serialVersionUID = 1L; + + public int length; + + public void validateParams() throws SerDeException { + if (length < 1) { + throw new SerDeException("VARCHAR length must be non-negative"); + } + } + + public void populateParams(String[] params) throws SerDeException { + if (params.length != 1) { + throw new SerDeException("Invalid number of parameters for VARCHAR"); + } + try { + length = Integer.valueOf(params[0]); + } catch (NumberFormatException err) { + throw new SerDeException("Error setting VARCHAR length: " + err); + } + } + + public String toString() { + StringBuffer sb = new StringBuffer(); + sb.append("("); + sb.append(length); + sb.append(")"); + return sb.toString(); + } + + @Override + public void readFields(DataInput in) throws IOException { + length = WritableUtils.readVInt(in); + try { + validateParams(); + } catch (SerDeException err) { + throw new IOException(err); + } + } + + @Override + public void write(DataOutput out) throws IOException { + WritableUtils.writeVInt(out, length); + } + + public int getLength() { + return length; + } + + public void setLength(int len) { + length = len; + } + + public boolean hasCharacterMaximumLength() { + return true; + } + public int getCharacterMaximumLength() { + return length; + } +} diff --git serde/src/test/org/apache/hadoop/hive/serde2/typeinfo/TestTypeInfoUtils.java serde/src/test/org/apache/hadoop/hive/serde2/typeinfo/TestTypeInfoUtils.java new file mode 100644 index 0000000..798987c --- /dev/null +++ serde/src/test/org/apache/hadoop/hive/serde2/typeinfo/TestTypeInfoUtils.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.typeinfo; + +import junit.framework.TestCase; + +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; + +public class TestTypeInfoUtils extends TestCase { + + static void parseTypeString(String typeString, boolean exceptionExpected) { + boolean caughtException = false; + try { + TypeInfoUtils.getTypeInfoFromTypeString(typeString); + } catch (IllegalArgumentException err) { + caughtException = true; + } + assertEquals("parsing typestring " + typeString, exceptionExpected, caughtException); + } + + public void testTypeInfoParser() { + String[] validTypeStrings = { + "int", + "string", + "varchar(10)", + "array" + }; + + String[] invalidTypeStrings = { + "array<", + "varchar(123", + "varchar(123,", + "varchar()", + "varchar(" + }; + + for (String typeString : validTypeStrings) { + parseTypeString(typeString, false); + } + for (String typeString : invalidTypeStrings) { + parseTypeString(typeString, true); + } + } + + public void testVarcharNoParams() { + boolean caughtException = false; + try { + TypeInfoUtils.getTypeInfoFromTypeString("varchar"); + } catch (Exception err) { + caughtException = true; + } + assertEquals("varchar TypeInfo with no params should fail", true, caughtException); + } +} diff --git service/if/TCLIService.thrift service/if/TCLIService.thrift index 8dc2a90..318595e 100644 --- service/if/TCLIService.thrift +++ service/if/TCLIService.thrift @@ -59,7 +59,8 @@ enum TTypeId { USER_DEFINED_TYPE, DECIMAL_TYPE, NULL_TYPE, - DATE_TYPE + DATE_TYPE, + VARCHAR_TYPE } const set PRIMITIVE_TYPES = [ @@ -76,6 +77,7 @@ const set PRIMITIVE_TYPES = [ TTypeId.DECIMAL_TYPE, TTypeId.NULL_TYPE TTypeId.DATE_TYPE + TTypeId.VARCHAR_TYPE ] const set COMPLEX_TYPES = [ @@ -109,6 +111,7 @@ const map TYPE_NAMES = { TTypeId.DECIMAL_TYPE: "DECIMAL", TTypeId.NULL_TYPE: "NULL" TTypeId.DATE_TYPE: "DATE" + TTypeId.VARCHAR_TYPE: "VARCHAR" } // Thrift does not support recursively defined types or forward declarations, @@ -156,11 +159,25 @@ const map TYPE_NAMES = { typedef i32 TTypeEntryPtr +// Valid TTypeQualifiers key names +const string CHARACTER_MAXIMUM_LENGTH = "characterMaximumLength" + +union TTypeQualifierValue { + 1: optional i32 i32Value + 2: optional string stringValue +} + +// Type qualifiers for primitive type. +struct TTypeQualifiers { + 1: required map qualifiers +} + // Type entry for a primitive type. struct TPrimitiveTypeEntry { // The primitive type token. This must satisfy the condition // that type is in the PRIMITIVE_TYPES set. 1: required TTypeId type + 2: optional TTypeQualifiers typeQualifiers } // Type entry for an ARRAY type. diff --git service/src/gen/thrift/gen-cpp/TCLIService_constants.cpp service/src/gen/thrift/gen-cpp/TCLIService_constants.cpp index 7284cc5..d085b30 100644 --- service/src/gen/thrift/gen-cpp/TCLIService_constants.cpp +++ service/src/gen/thrift/gen-cpp/TCLIService_constants.cpp @@ -24,6 +24,7 @@ TCLIServiceConstants::TCLIServiceConstants() { PRIMITIVE_TYPES.insert((TTypeId::type)15); PRIMITIVE_TYPES.insert((TTypeId::type)16); PRIMITIVE_TYPES.insert((TTypeId::type)17); + PRIMITIVE_TYPES.insert((TTypeId::type)18); COMPLEX_TYPES.insert((TTypeId::type)10); COMPLEX_TYPES.insert((TTypeId::type)11); @@ -51,6 +52,9 @@ TCLIServiceConstants::TCLIServiceConstants() { TYPE_NAMES.insert(std::make_pair((TTypeId::type)15, "DECIMAL")); TYPE_NAMES.insert(std::make_pair((TTypeId::type)16, "NULL")); TYPE_NAMES.insert(std::make_pair((TTypeId::type)17, "DATE")); + TYPE_NAMES.insert(std::make_pair((TTypeId::type)18, "VARCHAR")); + + CHARACTER_MAXIMUM_LENGTH = "characterMaximumLength"; } diff --git service/src/gen/thrift/gen-cpp/TCLIService_constants.h service/src/gen/thrift/gen-cpp/TCLIService_constants.h index cd63739..7471811 100644 --- service/src/gen/thrift/gen-cpp/TCLIService_constants.h +++ service/src/gen/thrift/gen-cpp/TCLIService_constants.h @@ -19,6 +19,7 @@ class TCLIServiceConstants { std::set COMPLEX_TYPES; std::set COLLECTION_TYPES; std::map TYPE_NAMES; + std::string CHARACTER_MAXIMUM_LENGTH; }; extern const TCLIServiceConstants g_TCLIService_constants; diff --git service/src/gen/thrift/gen-cpp/TCLIService_types.cpp service/src/gen/thrift/gen-cpp/TCLIService_types.cpp index f9c9fb9..187bb74 100644 --- service/src/gen/thrift/gen-cpp/TCLIService_types.cpp +++ service/src/gen/thrift/gen-cpp/TCLIService_types.cpp @@ -36,7 +36,8 @@ int _kTTypeIdValues[] = { TTypeId::USER_DEFINED_TYPE, TTypeId::DECIMAL_TYPE, TTypeId::NULL_TYPE, - TTypeId::DATE_TYPE + TTypeId::DATE_TYPE, + TTypeId::VARCHAR_TYPE }; const char* _kTTypeIdNames[] = { "BOOLEAN_TYPE", @@ -56,9 +57,10 @@ const char* _kTTypeIdNames[] = { "USER_DEFINED_TYPE", "DECIMAL_TYPE", "NULL_TYPE", - "DATE_TYPE" + "DATE_TYPE", + "VARCHAR_TYPE" }; -const std::map _TTypeId_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(18, _kTTypeIdValues, _kTTypeIdNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL)); +const std::map _TTypeId_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(19, _kTTypeIdValues, _kTTypeIdNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL)); int _kTStatusCodeValues[] = { TStatusCode::SUCCESS_STATUS, @@ -238,8 +240,173 @@ const char* _kTFetchOrientationNames[] = { }; const std::map _TFetchOrientation_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(6, _kTFetchOrientationValues, _kTFetchOrientationNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL)); -const char* TPrimitiveTypeEntry::ascii_fingerprint = "8BBB3D0C3B370CB38F2D1340BB79F0AA"; -const uint8_t TPrimitiveTypeEntry::binary_fingerprint[16] = {0x8B,0xBB,0x3D,0x0C,0x3B,0x37,0x0C,0xB3,0x8F,0x2D,0x13,0x40,0xBB,0x79,0xF0,0xAA}; +const char* TTypeQualifierValue::ascii_fingerprint = "A7801670116150C65ACA43E6F679BA79"; +const uint8_t TTypeQualifierValue::binary_fingerprint[16] = {0xA7,0x80,0x16,0x70,0x11,0x61,0x50,0xC6,0x5A,0xCA,0x43,0xE6,0xF6,0x79,0xBA,0x79}; + +uint32_t TTypeQualifierValue::read(::apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->i32Value); + this->__isset.i32Value = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->stringValue); + this->__isset.stringValue = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t TTypeQualifierValue::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("TTypeQualifierValue"); + + if (this->__isset.i32Value) { + xfer += oprot->writeFieldBegin("i32Value", ::apache::thrift::protocol::T_I32, 1); + xfer += oprot->writeI32(this->i32Value); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.stringValue) { + xfer += oprot->writeFieldBegin("stringValue", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeString(this->stringValue); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(TTypeQualifierValue &a, TTypeQualifierValue &b) { + using ::std::swap; + swap(a.i32Value, b.i32Value); + swap(a.stringValue, b.stringValue); + swap(a.__isset, b.__isset); +} + +const char* TTypeQualifiers::ascii_fingerprint = "6C72981CFA989214285648FA8C196C47"; +const uint8_t TTypeQualifiers::binary_fingerprint[16] = {0x6C,0x72,0x98,0x1C,0xFA,0x98,0x92,0x14,0x28,0x56,0x48,0xFA,0x8C,0x19,0x6C,0x47}; + +uint32_t TTypeQualifiers::read(::apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + bool isset_qualifiers = false; + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_MAP) { + { + this->qualifiers.clear(); + uint32_t _size0; + ::apache::thrift::protocol::TType _ktype1; + ::apache::thrift::protocol::TType _vtype2; + xfer += iprot->readMapBegin(_ktype1, _vtype2, _size0); + uint32_t _i4; + for (_i4 = 0; _i4 < _size0; ++_i4) + { + std::string _key5; + xfer += iprot->readString(_key5); + TTypeQualifierValue& _val6 = this->qualifiers[_key5]; + xfer += _val6.read(iprot); + } + xfer += iprot->readMapEnd(); + } + isset_qualifiers = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + if (!isset_qualifiers) + throw TProtocolException(TProtocolException::INVALID_DATA); + return xfer; +} + +uint32_t TTypeQualifiers::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("TTypeQualifiers"); + + xfer += oprot->writeFieldBegin("qualifiers", ::apache::thrift::protocol::T_MAP, 1); + { + xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_STRUCT, static_cast(this->qualifiers.size())); + std::map ::const_iterator _iter7; + for (_iter7 = this->qualifiers.begin(); _iter7 != this->qualifiers.end(); ++_iter7) + { + xfer += oprot->writeString(_iter7->first); + xfer += _iter7->second.write(oprot); + } + xfer += oprot->writeMapEnd(); + } + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(TTypeQualifiers &a, TTypeQualifiers &b) { + using ::std::swap; + swap(a.qualifiers, b.qualifiers); +} + +const char* TPrimitiveTypeEntry::ascii_fingerprint = "755674F6A5C8EB47868686AE386FBC1C"; +const uint8_t TPrimitiveTypeEntry::binary_fingerprint[16] = {0x75,0x56,0x74,0xF6,0xA5,0xC8,0xEB,0x47,0x86,0x86,0x86,0xAE,0x38,0x6F,0xBC,0x1C}; uint32_t TPrimitiveTypeEntry::read(::apache::thrift::protocol::TProtocol* iprot) { @@ -264,14 +431,22 @@ uint32_t TPrimitiveTypeEntry::read(::apache::thrift::protocol::TProtocol* iprot) { case 1: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast0; - xfer += iprot->readI32(ecast0); - this->type = (TTypeId::type)ecast0; + int32_t ecast8; + xfer += iprot->readI32(ecast8); + this->type = (TTypeId::type)ecast8; isset_type = true; } else { xfer += iprot->skip(ftype); } break; + case 2: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->typeQualifiers.read(iprot); + this->__isset.typeQualifiers = true; + } else { + xfer += iprot->skip(ftype); + } + break; default: xfer += iprot->skip(ftype); break; @@ -294,6 +469,11 @@ uint32_t TPrimitiveTypeEntry::write(::apache::thrift::protocol::TProtocol* oprot xfer += oprot->writeI32((int32_t)this->type); xfer += oprot->writeFieldEnd(); + if (this->__isset.typeQualifiers) { + xfer += oprot->writeFieldBegin("typeQualifiers", ::apache::thrift::protocol::T_STRUCT, 2); + xfer += this->typeQualifiers.write(oprot); + xfer += oprot->writeFieldEnd(); + } xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -302,6 +482,8 @@ uint32_t TPrimitiveTypeEntry::write(::apache::thrift::protocol::TProtocol* oprot void swap(TPrimitiveTypeEntry &a, TPrimitiveTypeEntry &b) { using ::std::swap; swap(a.type, b.type); + swap(a.typeQualifiers, b.typeQualifiers); + swap(a.__isset, b.__isset); } const char* TArrayTypeEntry::ascii_fingerprint = "E86CACEB22240450EDCBEFC3A83970E4"; @@ -476,17 +658,17 @@ uint32_t TStructTypeEntry::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_MAP) { { this->nameToTypePtr.clear(); - uint32_t _size1; - ::apache::thrift::protocol::TType _ktype2; - ::apache::thrift::protocol::TType _vtype3; - xfer += iprot->readMapBegin(_ktype2, _vtype3, _size1); - uint32_t _i5; - for (_i5 = 0; _i5 < _size1; ++_i5) + uint32_t _size9; + ::apache::thrift::protocol::TType _ktype10; + ::apache::thrift::protocol::TType _vtype11; + xfer += iprot->readMapBegin(_ktype10, _vtype11, _size9); + uint32_t _i13; + for (_i13 = 0; _i13 < _size9; ++_i13) { - std::string _key6; - xfer += iprot->readString(_key6); - TTypeEntryPtr& _val7 = this->nameToTypePtr[_key6]; - xfer += iprot->readI32(_val7); + std::string _key14; + xfer += iprot->readString(_key14); + TTypeEntryPtr& _val15 = this->nameToTypePtr[_key14]; + xfer += iprot->readI32(_val15); } xfer += iprot->readMapEnd(); } @@ -516,11 +698,11 @@ uint32_t TStructTypeEntry::write(::apache::thrift::protocol::TProtocol* oprot) c xfer += oprot->writeFieldBegin("nameToTypePtr", ::apache::thrift::protocol::T_MAP, 1); { xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_I32, static_cast(this->nameToTypePtr.size())); - std::map ::const_iterator _iter8; - for (_iter8 = this->nameToTypePtr.begin(); _iter8 != this->nameToTypePtr.end(); ++_iter8) + std::map ::const_iterator _iter16; + for (_iter16 = this->nameToTypePtr.begin(); _iter16 != this->nameToTypePtr.end(); ++_iter16) { - xfer += oprot->writeString(_iter8->first); - xfer += oprot->writeI32(_iter8->second); + xfer += oprot->writeString(_iter16->first); + xfer += oprot->writeI32(_iter16->second); } xfer += oprot->writeMapEnd(); } @@ -564,17 +746,17 @@ uint32_t TUnionTypeEntry::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_MAP) { { this->nameToTypePtr.clear(); - uint32_t _size9; - ::apache::thrift::protocol::TType _ktype10; - ::apache::thrift::protocol::TType _vtype11; - xfer += iprot->readMapBegin(_ktype10, _vtype11, _size9); - uint32_t _i13; - for (_i13 = 0; _i13 < _size9; ++_i13) + uint32_t _size17; + ::apache::thrift::protocol::TType _ktype18; + ::apache::thrift::protocol::TType _vtype19; + xfer += iprot->readMapBegin(_ktype18, _vtype19, _size17); + uint32_t _i21; + for (_i21 = 0; _i21 < _size17; ++_i21) { - std::string _key14; - xfer += iprot->readString(_key14); - TTypeEntryPtr& _val15 = this->nameToTypePtr[_key14]; - xfer += iprot->readI32(_val15); + std::string _key22; + xfer += iprot->readString(_key22); + TTypeEntryPtr& _val23 = this->nameToTypePtr[_key22]; + xfer += iprot->readI32(_val23); } xfer += iprot->readMapEnd(); } @@ -604,11 +786,11 @@ uint32_t TUnionTypeEntry::write(::apache::thrift::protocol::TProtocol* oprot) co xfer += oprot->writeFieldBegin("nameToTypePtr", ::apache::thrift::protocol::T_MAP, 1); { xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_I32, static_cast(this->nameToTypePtr.size())); - std::map ::const_iterator _iter16; - for (_iter16 = this->nameToTypePtr.begin(); _iter16 != this->nameToTypePtr.end(); ++_iter16) + std::map ::const_iterator _iter24; + for (_iter24 = this->nameToTypePtr.begin(); _iter24 != this->nameToTypePtr.end(); ++_iter24) { - xfer += oprot->writeString(_iter16->first); - xfer += oprot->writeI32(_iter16->second); + xfer += oprot->writeString(_iter24->first); + xfer += oprot->writeI32(_iter24->second); } xfer += oprot->writeMapEnd(); } @@ -688,8 +870,8 @@ void swap(TUserDefinedTypeEntry &a, TUserDefinedTypeEntry &b) { swap(a.typeClassName, b.typeClassName); } -const char* TTypeEntry::ascii_fingerprint = "0C3ACE4054603E2D37B8BFEBA79F4159"; -const uint8_t TTypeEntry::binary_fingerprint[16] = {0x0C,0x3A,0xCE,0x40,0x54,0x60,0x3E,0x2D,0x37,0xB8,0xBF,0xEB,0xA7,0x9F,0x41,0x59}; +const char* TTypeEntry::ascii_fingerprint = "2FE56D9097E325DAA7E933738C6D325F"; +const uint8_t TTypeEntry::binary_fingerprint[16] = {0x2F,0xE5,0x6D,0x90,0x97,0xE3,0x25,0xDA,0xA7,0xE9,0x33,0x73,0x8C,0x6D,0x32,0x5F}; uint32_t TTypeEntry::read(::apache::thrift::protocol::TProtocol* iprot) { @@ -815,8 +997,8 @@ void swap(TTypeEntry &a, TTypeEntry &b) { swap(a.__isset, b.__isset); } -const char* TTypeDesc::ascii_fingerprint = "60CA5B8BACFCD38D1D3EC1F0E3F1C36A"; -const uint8_t TTypeDesc::binary_fingerprint[16] = {0x60,0xCA,0x5B,0x8B,0xAC,0xFC,0xD3,0x8D,0x1D,0x3E,0xC1,0xF0,0xE3,0xF1,0xC3,0x6A}; +const char* TTypeDesc::ascii_fingerprint = "90B3C5A0B73419A84E85E0E48C452AA5"; +const uint8_t TTypeDesc::binary_fingerprint[16] = {0x90,0xB3,0xC5,0xA0,0xB7,0x34,0x19,0xA8,0x4E,0x85,0xE0,0xE4,0x8C,0x45,0x2A,0xA5}; uint32_t TTypeDesc::read(::apache::thrift::protocol::TProtocol* iprot) { @@ -843,14 +1025,14 @@ uint32_t TTypeDesc::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->types.clear(); - uint32_t _size17; - ::apache::thrift::protocol::TType _etype20; - xfer += iprot->readListBegin(_etype20, _size17); - this->types.resize(_size17); - uint32_t _i21; - for (_i21 = 0; _i21 < _size17; ++_i21) + uint32_t _size25; + ::apache::thrift::protocol::TType _etype28; + xfer += iprot->readListBegin(_etype28, _size25); + this->types.resize(_size25); + uint32_t _i29; + for (_i29 = 0; _i29 < _size25; ++_i29) { - xfer += this->types[_i21].read(iprot); + xfer += this->types[_i29].read(iprot); } xfer += iprot->readListEnd(); } @@ -880,10 +1062,10 @@ uint32_t TTypeDesc::write(::apache::thrift::protocol::TProtocol* oprot) const { xfer += oprot->writeFieldBegin("types", ::apache::thrift::protocol::T_LIST, 1); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->types.size())); - std::vector ::const_iterator _iter22; - for (_iter22 = this->types.begin(); _iter22 != this->types.end(); ++_iter22) + std::vector ::const_iterator _iter30; + for (_iter30 = this->types.begin(); _iter30 != this->types.end(); ++_iter30) { - xfer += (*_iter22).write(oprot); + xfer += (*_iter30).write(oprot); } xfer += oprot->writeListEnd(); } @@ -899,8 +1081,8 @@ void swap(TTypeDesc &a, TTypeDesc &b) { swap(a.types, b.types); } -const char* TColumnDesc::ascii_fingerprint = "0DF9A37B81B1EE73D35A0AC01F33A48D"; -const uint8_t TColumnDesc::binary_fingerprint[16] = {0x0D,0xF9,0xA3,0x7B,0x81,0xB1,0xEE,0x73,0xD3,0x5A,0x0A,0xC0,0x1F,0x33,0xA4,0x8D}; +const char* TColumnDesc::ascii_fingerprint = "EABED9009D5FCABFCA65612069F2A849"; +const uint8_t TColumnDesc::binary_fingerprint[16] = {0xEA,0xBE,0xD9,0x00,0x9D,0x5F,0xCA,0xBF,0xCA,0x65,0x61,0x20,0x69,0xF2,0xA8,0x49}; uint32_t TColumnDesc::read(::apache::thrift::protocol::TProtocol* iprot) { @@ -1010,8 +1192,8 @@ void swap(TColumnDesc &a, TColumnDesc &b) { swap(a.__isset, b.__isset); } -const char* TTableSchema::ascii_fingerprint = "E67E789F1EF836E4B9FC922C788AFDC8"; -const uint8_t TTableSchema::binary_fingerprint[16] = {0xE6,0x7E,0x78,0x9F,0x1E,0xF8,0x36,0xE4,0xB9,0xFC,0x92,0x2C,0x78,0x8A,0xFD,0xC8}; +const char* TTableSchema::ascii_fingerprint = "7A1811E49313E5977107FC667B20E39D"; +const uint8_t TTableSchema::binary_fingerprint[16] = {0x7A,0x18,0x11,0xE4,0x93,0x13,0xE5,0x97,0x71,0x07,0xFC,0x66,0x7B,0x20,0xE3,0x9D}; uint32_t TTableSchema::read(::apache::thrift::protocol::TProtocol* iprot) { @@ -1038,14 +1220,14 @@ uint32_t TTableSchema::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->columns.clear(); - uint32_t _size23; - ::apache::thrift::protocol::TType _etype26; - xfer += iprot->readListBegin(_etype26, _size23); - this->columns.resize(_size23); - uint32_t _i27; - for (_i27 = 0; _i27 < _size23; ++_i27) + uint32_t _size31; + ::apache::thrift::protocol::TType _etype34; + xfer += iprot->readListBegin(_etype34, _size31); + this->columns.resize(_size31); + uint32_t _i35; + for (_i35 = 0; _i35 < _size31; ++_i35) { - xfer += this->columns[_i27].read(iprot); + xfer += this->columns[_i35].read(iprot); } xfer += iprot->readListEnd(); } @@ -1075,10 +1257,10 @@ uint32_t TTableSchema::write(::apache::thrift::protocol::TProtocol* oprot) const xfer += oprot->writeFieldBegin("columns", ::apache::thrift::protocol::T_LIST, 1); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->columns.size())); - std::vector ::const_iterator _iter28; - for (_iter28 = this->columns.begin(); _iter28 != this->columns.end(); ++_iter28) + std::vector ::const_iterator _iter36; + for (_iter36 = this->columns.begin(); _iter36 != this->columns.end(); ++_iter36) { - xfer += (*_iter28).write(oprot); + xfer += (*_iter36).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1562,14 +1744,14 @@ uint32_t TColumn::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->boolColumn.clear(); - uint32_t _size29; - ::apache::thrift::protocol::TType _etype32; - xfer += iprot->readListBegin(_etype32, _size29); - this->boolColumn.resize(_size29); - uint32_t _i33; - for (_i33 = 0; _i33 < _size29; ++_i33) + uint32_t _size37; + ::apache::thrift::protocol::TType _etype40; + xfer += iprot->readListBegin(_etype40, _size37); + this->boolColumn.resize(_size37); + uint32_t _i41; + for (_i41 = 0; _i41 < _size37; ++_i41) { - xfer += this->boolColumn[_i33].read(iprot); + xfer += this->boolColumn[_i41].read(iprot); } xfer += iprot->readListEnd(); } @@ -1582,14 +1764,14 @@ uint32_t TColumn::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->byteColumn.clear(); - uint32_t _size34; - ::apache::thrift::protocol::TType _etype37; - xfer += iprot->readListBegin(_etype37, _size34); - this->byteColumn.resize(_size34); - uint32_t _i38; - for (_i38 = 0; _i38 < _size34; ++_i38) + uint32_t _size42; + ::apache::thrift::protocol::TType _etype45; + xfer += iprot->readListBegin(_etype45, _size42); + this->byteColumn.resize(_size42); + uint32_t _i46; + for (_i46 = 0; _i46 < _size42; ++_i46) { - xfer += this->byteColumn[_i38].read(iprot); + xfer += this->byteColumn[_i46].read(iprot); } xfer += iprot->readListEnd(); } @@ -1602,14 +1784,14 @@ uint32_t TColumn::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->i16Column.clear(); - uint32_t _size39; - ::apache::thrift::protocol::TType _etype42; - xfer += iprot->readListBegin(_etype42, _size39); - this->i16Column.resize(_size39); - uint32_t _i43; - for (_i43 = 0; _i43 < _size39; ++_i43) + uint32_t _size47; + ::apache::thrift::protocol::TType _etype50; + xfer += iprot->readListBegin(_etype50, _size47); + this->i16Column.resize(_size47); + uint32_t _i51; + for (_i51 = 0; _i51 < _size47; ++_i51) { - xfer += this->i16Column[_i43].read(iprot); + xfer += this->i16Column[_i51].read(iprot); } xfer += iprot->readListEnd(); } @@ -1622,14 +1804,14 @@ uint32_t TColumn::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->i32Column.clear(); - uint32_t _size44; - ::apache::thrift::protocol::TType _etype47; - xfer += iprot->readListBegin(_etype47, _size44); - this->i32Column.resize(_size44); - uint32_t _i48; - for (_i48 = 0; _i48 < _size44; ++_i48) + uint32_t _size52; + ::apache::thrift::protocol::TType _etype55; + xfer += iprot->readListBegin(_etype55, _size52); + this->i32Column.resize(_size52); + uint32_t _i56; + for (_i56 = 0; _i56 < _size52; ++_i56) { - xfer += this->i32Column[_i48].read(iprot); + xfer += this->i32Column[_i56].read(iprot); } xfer += iprot->readListEnd(); } @@ -1642,14 +1824,14 @@ uint32_t TColumn::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->i64Column.clear(); - uint32_t _size49; - ::apache::thrift::protocol::TType _etype52; - xfer += iprot->readListBegin(_etype52, _size49); - this->i64Column.resize(_size49); - uint32_t _i53; - for (_i53 = 0; _i53 < _size49; ++_i53) + uint32_t _size57; + ::apache::thrift::protocol::TType _etype60; + xfer += iprot->readListBegin(_etype60, _size57); + this->i64Column.resize(_size57); + uint32_t _i61; + for (_i61 = 0; _i61 < _size57; ++_i61) { - xfer += this->i64Column[_i53].read(iprot); + xfer += this->i64Column[_i61].read(iprot); } xfer += iprot->readListEnd(); } @@ -1662,14 +1844,14 @@ uint32_t TColumn::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->doubleColumn.clear(); - uint32_t _size54; - ::apache::thrift::protocol::TType _etype57; - xfer += iprot->readListBegin(_etype57, _size54); - this->doubleColumn.resize(_size54); - uint32_t _i58; - for (_i58 = 0; _i58 < _size54; ++_i58) + uint32_t _size62; + ::apache::thrift::protocol::TType _etype65; + xfer += iprot->readListBegin(_etype65, _size62); + this->doubleColumn.resize(_size62); + uint32_t _i66; + for (_i66 = 0; _i66 < _size62; ++_i66) { - xfer += this->doubleColumn[_i58].read(iprot); + xfer += this->doubleColumn[_i66].read(iprot); } xfer += iprot->readListEnd(); } @@ -1682,14 +1864,14 @@ uint32_t TColumn::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->stringColumn.clear(); - uint32_t _size59; - ::apache::thrift::protocol::TType _etype62; - xfer += iprot->readListBegin(_etype62, _size59); - this->stringColumn.resize(_size59); - uint32_t _i63; - for (_i63 = 0; _i63 < _size59; ++_i63) + uint32_t _size67; + ::apache::thrift::protocol::TType _etype70; + xfer += iprot->readListBegin(_etype70, _size67); + this->stringColumn.resize(_size67); + uint32_t _i71; + for (_i71 = 0; _i71 < _size67; ++_i71) { - xfer += this->stringColumn[_i63].read(iprot); + xfer += this->stringColumn[_i71].read(iprot); } xfer += iprot->readListEnd(); } @@ -1717,10 +1899,10 @@ uint32_t TColumn::write(::apache::thrift::protocol::TProtocol* oprot) const { xfer += oprot->writeFieldBegin("boolColumn", ::apache::thrift::protocol::T_LIST, 1); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->boolColumn.size())); - std::vector ::const_iterator _iter64; - for (_iter64 = this->boolColumn.begin(); _iter64 != this->boolColumn.end(); ++_iter64) + std::vector ::const_iterator _iter72; + for (_iter72 = this->boolColumn.begin(); _iter72 != this->boolColumn.end(); ++_iter72) { - xfer += (*_iter64).write(oprot); + xfer += (*_iter72).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1729,10 +1911,10 @@ uint32_t TColumn::write(::apache::thrift::protocol::TProtocol* oprot) const { xfer += oprot->writeFieldBegin("byteColumn", ::apache::thrift::protocol::T_LIST, 2); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->byteColumn.size())); - std::vector ::const_iterator _iter65; - for (_iter65 = this->byteColumn.begin(); _iter65 != this->byteColumn.end(); ++_iter65) + std::vector ::const_iterator _iter73; + for (_iter73 = this->byteColumn.begin(); _iter73 != this->byteColumn.end(); ++_iter73) { - xfer += (*_iter65).write(oprot); + xfer += (*_iter73).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1741,10 +1923,10 @@ uint32_t TColumn::write(::apache::thrift::protocol::TProtocol* oprot) const { xfer += oprot->writeFieldBegin("i16Column", ::apache::thrift::protocol::T_LIST, 3); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->i16Column.size())); - std::vector ::const_iterator _iter66; - for (_iter66 = this->i16Column.begin(); _iter66 != this->i16Column.end(); ++_iter66) + std::vector ::const_iterator _iter74; + for (_iter74 = this->i16Column.begin(); _iter74 != this->i16Column.end(); ++_iter74) { - xfer += (*_iter66).write(oprot); + xfer += (*_iter74).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1753,10 +1935,10 @@ uint32_t TColumn::write(::apache::thrift::protocol::TProtocol* oprot) const { xfer += oprot->writeFieldBegin("i32Column", ::apache::thrift::protocol::T_LIST, 4); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->i32Column.size())); - std::vector ::const_iterator _iter67; - for (_iter67 = this->i32Column.begin(); _iter67 != this->i32Column.end(); ++_iter67) + std::vector ::const_iterator _iter75; + for (_iter75 = this->i32Column.begin(); _iter75 != this->i32Column.end(); ++_iter75) { - xfer += (*_iter67).write(oprot); + xfer += (*_iter75).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1765,10 +1947,10 @@ uint32_t TColumn::write(::apache::thrift::protocol::TProtocol* oprot) const { xfer += oprot->writeFieldBegin("i64Column", ::apache::thrift::protocol::T_LIST, 5); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->i64Column.size())); - std::vector ::const_iterator _iter68; - for (_iter68 = this->i64Column.begin(); _iter68 != this->i64Column.end(); ++_iter68) + std::vector ::const_iterator _iter76; + for (_iter76 = this->i64Column.begin(); _iter76 != this->i64Column.end(); ++_iter76) { - xfer += (*_iter68).write(oprot); + xfer += (*_iter76).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1777,10 +1959,10 @@ uint32_t TColumn::write(::apache::thrift::protocol::TProtocol* oprot) const { xfer += oprot->writeFieldBegin("doubleColumn", ::apache::thrift::protocol::T_LIST, 6); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->doubleColumn.size())); - std::vector ::const_iterator _iter69; - for (_iter69 = this->doubleColumn.begin(); _iter69 != this->doubleColumn.end(); ++_iter69) + std::vector ::const_iterator _iter77; + for (_iter77 = this->doubleColumn.begin(); _iter77 != this->doubleColumn.end(); ++_iter77) { - xfer += (*_iter69).write(oprot); + xfer += (*_iter77).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1789,10 +1971,10 @@ uint32_t TColumn::write(::apache::thrift::protocol::TProtocol* oprot) const { xfer += oprot->writeFieldBegin("stringColumn", ::apache::thrift::protocol::T_LIST, 7); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->stringColumn.size())); - std::vector ::const_iterator _iter70; - for (_iter70 = this->stringColumn.begin(); _iter70 != this->stringColumn.end(); ++_iter70) + std::vector ::const_iterator _iter78; + for (_iter78 = this->stringColumn.begin(); _iter78 != this->stringColumn.end(); ++_iter78) { - xfer += (*_iter70).write(oprot); + xfer += (*_iter78).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1983,14 +2165,14 @@ uint32_t TRow::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->colVals.clear(); - uint32_t _size71; - ::apache::thrift::protocol::TType _etype74; - xfer += iprot->readListBegin(_etype74, _size71); - this->colVals.resize(_size71); - uint32_t _i75; - for (_i75 = 0; _i75 < _size71; ++_i75) + uint32_t _size79; + ::apache::thrift::protocol::TType _etype82; + xfer += iprot->readListBegin(_etype82, _size79); + this->colVals.resize(_size79); + uint32_t _i83; + for (_i83 = 0; _i83 < _size79; ++_i83) { - xfer += this->colVals[_i75].read(iprot); + xfer += this->colVals[_i83].read(iprot); } xfer += iprot->readListEnd(); } @@ -2020,10 +2202,10 @@ uint32_t TRow::write(::apache::thrift::protocol::TProtocol* oprot) const { xfer += oprot->writeFieldBegin("colVals", ::apache::thrift::protocol::T_LIST, 1); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->colVals.size())); - std::vector ::const_iterator _iter76; - for (_iter76 = this->colVals.begin(); _iter76 != this->colVals.end(); ++_iter76) + std::vector ::const_iterator _iter84; + for (_iter84 = this->colVals.begin(); _iter84 != this->colVals.end(); ++_iter84) { - xfer += (*_iter76).write(oprot); + xfer += (*_iter84).write(oprot); } xfer += oprot->writeListEnd(); } @@ -2076,14 +2258,14 @@ uint32_t TRowSet::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->rows.clear(); - uint32_t _size77; - ::apache::thrift::protocol::TType _etype80; - xfer += iprot->readListBegin(_etype80, _size77); - this->rows.resize(_size77); - uint32_t _i81; - for (_i81 = 0; _i81 < _size77; ++_i81) + uint32_t _size85; + ::apache::thrift::protocol::TType _etype88; + xfer += iprot->readListBegin(_etype88, _size85); + this->rows.resize(_size85); + uint32_t _i89; + for (_i89 = 0; _i89 < _size85; ++_i89) { - xfer += this->rows[_i81].read(iprot); + xfer += this->rows[_i89].read(iprot); } xfer += iprot->readListEnd(); } @@ -2096,14 +2278,14 @@ uint32_t TRowSet::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->columns.clear(); - uint32_t _size82; - ::apache::thrift::protocol::TType _etype85; - xfer += iprot->readListBegin(_etype85, _size82); - this->columns.resize(_size82); - uint32_t _i86; - for (_i86 = 0; _i86 < _size82; ++_i86) + uint32_t _size90; + ::apache::thrift::protocol::TType _etype93; + xfer += iprot->readListBegin(_etype93, _size90); + this->columns.resize(_size90); + uint32_t _i94; + for (_i94 = 0; _i94 < _size90; ++_i94) { - xfer += this->columns[_i86].read(iprot); + xfer += this->columns[_i94].read(iprot); } xfer += iprot->readListEnd(); } @@ -2139,10 +2321,10 @@ uint32_t TRowSet::write(::apache::thrift::protocol::TProtocol* oprot) const { xfer += oprot->writeFieldBegin("rows", ::apache::thrift::protocol::T_LIST, 2); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->rows.size())); - std::vector ::const_iterator _iter87; - for (_iter87 = this->rows.begin(); _iter87 != this->rows.end(); ++_iter87) + std::vector ::const_iterator _iter95; + for (_iter95 = this->rows.begin(); _iter95 != this->rows.end(); ++_iter95) { - xfer += (*_iter87).write(oprot); + xfer += (*_iter95).write(oprot); } xfer += oprot->writeListEnd(); } @@ -2152,10 +2334,10 @@ uint32_t TRowSet::write(::apache::thrift::protocol::TProtocol* oprot) const { xfer += oprot->writeFieldBegin("columns", ::apache::thrift::protocol::T_LIST, 3); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->columns.size())); - std::vector ::const_iterator _iter88; - for (_iter88 = this->columns.begin(); _iter88 != this->columns.end(); ++_iter88) + std::vector ::const_iterator _iter96; + for (_iter96 = this->columns.begin(); _iter96 != this->columns.end(); ++_iter96) { - xfer += (*_iter88).write(oprot); + xfer += (*_iter96).write(oprot); } xfer += oprot->writeListEnd(); } @@ -2200,9 +2382,9 @@ uint32_t TStatus::read(::apache::thrift::protocol::TProtocol* iprot) { { case 1: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast89; - xfer += iprot->readI32(ecast89); - this->statusCode = (TStatusCode::type)ecast89; + int32_t ecast97; + xfer += iprot->readI32(ecast97); + this->statusCode = (TStatusCode::type)ecast97; isset_statusCode = true; } else { xfer += iprot->skip(ftype); @@ -2212,14 +2394,14 @@ uint32_t TStatus::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->infoMessages.clear(); - uint32_t _size90; - ::apache::thrift::protocol::TType _etype93; - xfer += iprot->readListBegin(_etype93, _size90); - this->infoMessages.resize(_size90); - uint32_t _i94; - for (_i94 = 0; _i94 < _size90; ++_i94) + uint32_t _size98; + ::apache::thrift::protocol::TType _etype101; + xfer += iprot->readListBegin(_etype101, _size98); + this->infoMessages.resize(_size98); + uint32_t _i102; + for (_i102 = 0; _i102 < _size98; ++_i102) { - xfer += iprot->readString(this->infoMessages[_i94]); + xfer += iprot->readString(this->infoMessages[_i102]); } xfer += iprot->readListEnd(); } @@ -2278,10 +2460,10 @@ uint32_t TStatus::write(::apache::thrift::protocol::TProtocol* oprot) const { xfer += oprot->writeFieldBegin("infoMessages", ::apache::thrift::protocol::T_LIST, 2); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->infoMessages.size())); - std::vector ::const_iterator _iter95; - for (_iter95 = this->infoMessages.begin(); _iter95 != this->infoMessages.end(); ++_iter95) + std::vector ::const_iterator _iter103; + for (_iter103 = this->infoMessages.begin(); _iter103 != this->infoMessages.end(); ++_iter103) { - xfer += oprot->writeString((*_iter95)); + xfer += oprot->writeString((*_iter103)); } xfer += oprot->writeListEnd(); } @@ -2497,9 +2679,9 @@ uint32_t TOperationHandle::read(::apache::thrift::protocol::TProtocol* iprot) { break; case 2: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast96; - xfer += iprot->readI32(ecast96); - this->operationType = (TOperationType::type)ecast96; + int32_t ecast104; + xfer += iprot->readI32(ecast104); + this->operationType = (TOperationType::type)ecast104; isset_operationType = true; } else { xfer += iprot->skip(ftype); @@ -2600,9 +2782,9 @@ uint32_t TOpenSessionReq::read(::apache::thrift::protocol::TProtocol* iprot) { { case 1: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast97; - xfer += iprot->readI32(ecast97); - this->client_protocol = (TProtocolVersion::type)ecast97; + int32_t ecast105; + xfer += iprot->readI32(ecast105); + this->client_protocol = (TProtocolVersion::type)ecast105; isset_client_protocol = true; } else { xfer += iprot->skip(ftype); @@ -2628,17 +2810,17 @@ uint32_t TOpenSessionReq::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_MAP) { { this->configuration.clear(); - uint32_t _size98; - ::apache::thrift::protocol::TType _ktype99; - ::apache::thrift::protocol::TType _vtype100; - xfer += iprot->readMapBegin(_ktype99, _vtype100, _size98); - uint32_t _i102; - for (_i102 = 0; _i102 < _size98; ++_i102) + uint32_t _size106; + ::apache::thrift::protocol::TType _ktype107; + ::apache::thrift::protocol::TType _vtype108; + xfer += iprot->readMapBegin(_ktype107, _vtype108, _size106); + uint32_t _i110; + for (_i110 = 0; _i110 < _size106; ++_i110) { - std::string _key103; - xfer += iprot->readString(_key103); - std::string& _val104 = this->configuration[_key103]; - xfer += iprot->readString(_val104); + std::string _key111; + xfer += iprot->readString(_key111); + std::string& _val112 = this->configuration[_key111]; + xfer += iprot->readString(_val112); } xfer += iprot->readMapEnd(); } @@ -2683,11 +2865,11 @@ uint32_t TOpenSessionReq::write(::apache::thrift::protocol::TProtocol* oprot) co xfer += oprot->writeFieldBegin("configuration", ::apache::thrift::protocol::T_MAP, 4); { xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_STRING, static_cast(this->configuration.size())); - std::map ::const_iterator _iter105; - for (_iter105 = this->configuration.begin(); _iter105 != this->configuration.end(); ++_iter105) + std::map ::const_iterator _iter113; + for (_iter113 = this->configuration.begin(); _iter113 != this->configuration.end(); ++_iter113) { - xfer += oprot->writeString(_iter105->first); - xfer += oprot->writeString(_iter105->second); + xfer += oprot->writeString(_iter113->first); + xfer += oprot->writeString(_iter113->second); } xfer += oprot->writeMapEnd(); } @@ -2742,9 +2924,9 @@ uint32_t TOpenSessionResp::read(::apache::thrift::protocol::TProtocol* iprot) { break; case 2: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast106; - xfer += iprot->readI32(ecast106); - this->serverProtocolVersion = (TProtocolVersion::type)ecast106; + int32_t ecast114; + xfer += iprot->readI32(ecast114); + this->serverProtocolVersion = (TProtocolVersion::type)ecast114; isset_serverProtocolVersion = true; } else { xfer += iprot->skip(ftype); @@ -2762,17 +2944,17 @@ uint32_t TOpenSessionResp::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_MAP) { { this->configuration.clear(); - uint32_t _size107; - ::apache::thrift::protocol::TType _ktype108; - ::apache::thrift::protocol::TType _vtype109; - xfer += iprot->readMapBegin(_ktype108, _vtype109, _size107); - uint32_t _i111; - for (_i111 = 0; _i111 < _size107; ++_i111) + uint32_t _size115; + ::apache::thrift::protocol::TType _ktype116; + ::apache::thrift::protocol::TType _vtype117; + xfer += iprot->readMapBegin(_ktype116, _vtype117, _size115); + uint32_t _i119; + for (_i119 = 0; _i119 < _size115; ++_i119) { - std::string _key112; - xfer += iprot->readString(_key112); - std::string& _val113 = this->configuration[_key112]; - xfer += iprot->readString(_val113); + std::string _key120; + xfer += iprot->readString(_key120); + std::string& _val121 = this->configuration[_key120]; + xfer += iprot->readString(_val121); } xfer += iprot->readMapEnd(); } @@ -2818,11 +3000,11 @@ uint32_t TOpenSessionResp::write(::apache::thrift::protocol::TProtocol* oprot) c xfer += oprot->writeFieldBegin("configuration", ::apache::thrift::protocol::T_MAP, 4); { xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_STRING, static_cast(this->configuration.size())); - std::map ::const_iterator _iter114; - for (_iter114 = this->configuration.begin(); _iter114 != this->configuration.end(); ++_iter114) + std::map ::const_iterator _iter122; + for (_iter122 = this->configuration.begin(); _iter122 != this->configuration.end(); ++_iter122) { - xfer += oprot->writeString(_iter114->first); - xfer += oprot->writeString(_iter114->second); + xfer += oprot->writeString(_iter122->first); + xfer += oprot->writeString(_iter122->second); } xfer += oprot->writeMapEnd(); } @@ -3132,9 +3314,9 @@ uint32_t TGetInfoReq::read(::apache::thrift::protocol::TProtocol* iprot) { break; case 2: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast115; - xfer += iprot->readI32(ecast115); - this->infoType = (TGetInfoType::type)ecast115; + int32_t ecast123; + xfer += iprot->readI32(ecast123); + this->infoType = (TGetInfoType::type)ecast123; isset_infoType = true; } else { xfer += iprot->skip(ftype); @@ -3304,17 +3486,17 @@ uint32_t TExecuteStatementReq::read(::apache::thrift::protocol::TProtocol* iprot if (ftype == ::apache::thrift::protocol::T_MAP) { { this->confOverlay.clear(); - uint32_t _size116; - ::apache::thrift::protocol::TType _ktype117; - ::apache::thrift::protocol::TType _vtype118; - xfer += iprot->readMapBegin(_ktype117, _vtype118, _size116); - uint32_t _i120; - for (_i120 = 0; _i120 < _size116; ++_i120) + uint32_t _size124; + ::apache::thrift::protocol::TType _ktype125; + ::apache::thrift::protocol::TType _vtype126; + xfer += iprot->readMapBegin(_ktype125, _vtype126, _size124); + uint32_t _i128; + for (_i128 = 0; _i128 < _size124; ++_i128) { - std::string _key121; - xfer += iprot->readString(_key121); - std::string& _val122 = this->confOverlay[_key121]; - xfer += iprot->readString(_val122); + std::string _key129; + xfer += iprot->readString(_key129); + std::string& _val130 = this->confOverlay[_key129]; + xfer += iprot->readString(_val130); } xfer += iprot->readMapEnd(); } @@ -3355,11 +3537,11 @@ uint32_t TExecuteStatementReq::write(::apache::thrift::protocol::TProtocol* opro xfer += oprot->writeFieldBegin("confOverlay", ::apache::thrift::protocol::T_MAP, 3); { xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_STRING, static_cast(this->confOverlay.size())); - std::map ::const_iterator _iter123; - for (_iter123 = this->confOverlay.begin(); _iter123 != this->confOverlay.end(); ++_iter123) + std::map ::const_iterator _iter131; + for (_iter131 = this->confOverlay.begin(); _iter131 != this->confOverlay.end(); ++_iter131) { - xfer += oprot->writeString(_iter123->first); - xfer += oprot->writeString(_iter123->second); + xfer += oprot->writeString(_iter131->first); + xfer += oprot->writeString(_iter131->second); } xfer += oprot->writeMapEnd(); } @@ -3975,14 +4157,14 @@ uint32_t TGetTablesReq::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->tableTypes.clear(); - uint32_t _size124; - ::apache::thrift::protocol::TType _etype127; - xfer += iprot->readListBegin(_etype127, _size124); - this->tableTypes.resize(_size124); - uint32_t _i128; - for (_i128 = 0; _i128 < _size124; ++_i128) + uint32_t _size132; + ::apache::thrift::protocol::TType _etype135; + xfer += iprot->readListBegin(_etype135, _size132); + this->tableTypes.resize(_size132); + uint32_t _i136; + for (_i136 = 0; _i136 < _size132; ++_i136) { - xfer += iprot->readString(this->tableTypes[_i128]); + xfer += iprot->readString(this->tableTypes[_i136]); } xfer += iprot->readListEnd(); } @@ -4032,10 +4214,10 @@ uint32_t TGetTablesReq::write(::apache::thrift::protocol::TProtocol* oprot) cons xfer += oprot->writeFieldBegin("tableTypes", ::apache::thrift::protocol::T_LIST, 5); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->tableTypes.size())); - std::vector ::const_iterator _iter129; - for (_iter129 = this->tableTypes.begin(); _iter129 != this->tableTypes.end(); ++_iter129) + std::vector ::const_iterator _iter137; + for (_iter137 = this->tableTypes.begin(); _iter137 != this->tableTypes.end(); ++_iter137) { - xfer += oprot->writeString((*_iter129)); + xfer += oprot->writeString((*_iter137)); } xfer += oprot->writeListEnd(); } @@ -4764,9 +4946,9 @@ uint32_t TGetOperationStatusResp::read(::apache::thrift::protocol::TProtocol* ip break; case 2: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast130; - xfer += iprot->readI32(ecast130); - this->operationState = (TOperationState::type)ecast130; + int32_t ecast138; + xfer += iprot->readI32(ecast138); + this->operationState = (TOperationState::type)ecast138; this->__isset.operationState = true; } else { xfer += iprot->skip(ftype); @@ -5131,8 +5313,8 @@ void swap(TGetResultSetMetadataReq &a, TGetResultSetMetadataReq &b) { swap(a.operationHandle, b.operationHandle); } -const char* TGetResultSetMetadataResp::ascii_fingerprint = "8778316D0AFC17584F192162BFF2AEDE"; -const uint8_t TGetResultSetMetadataResp::binary_fingerprint[16] = {0x87,0x78,0x31,0x6D,0x0A,0xFC,0x17,0x58,0x4F,0x19,0x21,0x62,0xBF,0xF2,0xAE,0xDE}; +const char* TGetResultSetMetadataResp::ascii_fingerprint = "42CD49B7F49CC1B6D4E6F5FA2D7BE812"; +const uint8_t TGetResultSetMetadataResp::binary_fingerprint[16] = {0x42,0xCD,0x49,0xB7,0xF4,0x9C,0xC1,0xB6,0xD4,0xE6,0xF5,0xFA,0x2D,0x7B,0xE8,0x12}; uint32_t TGetResultSetMetadataResp::read(::apache::thrift::protocol::TProtocol* iprot) { @@ -5246,9 +5428,9 @@ uint32_t TFetchResultsReq::read(::apache::thrift::protocol::TProtocol* iprot) { break; case 2: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast131; - xfer += iprot->readI32(ecast131); - this->orientation = (TFetchOrientation::type)ecast131; + int32_t ecast139; + xfer += iprot->readI32(ecast139); + this->orientation = (TFetchOrientation::type)ecast139; isset_orientation = true; } else { xfer += iprot->skip(ftype); diff --git service/src/gen/thrift/gen-cpp/TCLIService_types.h service/src/gen/thrift/gen-cpp/TCLIService_types.h index f670179..18e72d5 100644 --- service/src/gen/thrift/gen-cpp/TCLIService_types.h +++ service/src/gen/thrift/gen-cpp/TCLIService_types.h @@ -43,7 +43,8 @@ struct TTypeId { USER_DEFINED_TYPE = 14, DECIMAL_TYPE = 15, NULL_TYPE = 16, - DATE_TYPE = 17 + DATE_TYPE = 17, + VARCHAR_TYPE = 18 }; }; @@ -166,12 +167,110 @@ typedef std::string TPattern; typedef std::string TPatternOrIdentifier; +typedef struct _TTypeQualifierValue__isset { + _TTypeQualifierValue__isset() : i32Value(false), stringValue(false) {} + bool i32Value; + bool stringValue; +} _TTypeQualifierValue__isset; + +class TTypeQualifierValue { + public: + + static const char* ascii_fingerprint; // = "A7801670116150C65ACA43E6F679BA79"; + static const uint8_t binary_fingerprint[16]; // = {0xA7,0x80,0x16,0x70,0x11,0x61,0x50,0xC6,0x5A,0xCA,0x43,0xE6,0xF6,0x79,0xBA,0x79}; + + TTypeQualifierValue() : i32Value(0), stringValue() { + } + + virtual ~TTypeQualifierValue() throw() {} + + int32_t i32Value; + std::string stringValue; + + _TTypeQualifierValue__isset __isset; + + void __set_i32Value(const int32_t val) { + i32Value = val; + __isset.i32Value = true; + } + + void __set_stringValue(const std::string& val) { + stringValue = val; + __isset.stringValue = true; + } + + bool operator == (const TTypeQualifierValue & rhs) const + { + if (__isset.i32Value != rhs.__isset.i32Value) + return false; + else if (__isset.i32Value && !(i32Value == rhs.i32Value)) + return false; + if (__isset.stringValue != rhs.__isset.stringValue) + return false; + else if (__isset.stringValue && !(stringValue == rhs.stringValue)) + return false; + return true; + } + bool operator != (const TTypeQualifierValue &rhs) const { + return !(*this == rhs); + } + + bool operator < (const TTypeQualifierValue & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +void swap(TTypeQualifierValue &a, TTypeQualifierValue &b); + + +class TTypeQualifiers { + public: + + static const char* ascii_fingerprint; // = "6C72981CFA989214285648FA8C196C47"; + static const uint8_t binary_fingerprint[16]; // = {0x6C,0x72,0x98,0x1C,0xFA,0x98,0x92,0x14,0x28,0x56,0x48,0xFA,0x8C,0x19,0x6C,0x47}; + + TTypeQualifiers() { + } + + virtual ~TTypeQualifiers() throw() {} + + std::map qualifiers; + + void __set_qualifiers(const std::map & val) { + qualifiers = val; + } + + bool operator == (const TTypeQualifiers & rhs) const + { + if (!(qualifiers == rhs.qualifiers)) + return false; + return true; + } + bool operator != (const TTypeQualifiers &rhs) const { + return !(*this == rhs); + } + + bool operator < (const TTypeQualifiers & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +void swap(TTypeQualifiers &a, TTypeQualifiers &b); + +typedef struct _TPrimitiveTypeEntry__isset { + _TPrimitiveTypeEntry__isset() : typeQualifiers(false) {} + bool typeQualifiers; +} _TPrimitiveTypeEntry__isset; class TPrimitiveTypeEntry { public: - static const char* ascii_fingerprint; // = "8BBB3D0C3B370CB38F2D1340BB79F0AA"; - static const uint8_t binary_fingerprint[16]; // = {0x8B,0xBB,0x3D,0x0C,0x3B,0x37,0x0C,0xB3,0x8F,0x2D,0x13,0x40,0xBB,0x79,0xF0,0xAA}; + static const char* ascii_fingerprint; // = "755674F6A5C8EB47868686AE386FBC1C"; + static const uint8_t binary_fingerprint[16]; // = {0x75,0x56,0x74,0xF6,0xA5,0xC8,0xEB,0x47,0x86,0x86,0x86,0xAE,0x38,0x6F,0xBC,0x1C}; TPrimitiveTypeEntry() : type((TTypeId::type)0) { } @@ -179,15 +278,27 @@ class TPrimitiveTypeEntry { virtual ~TPrimitiveTypeEntry() throw() {} TTypeId::type type; + TTypeQualifiers typeQualifiers; + + _TPrimitiveTypeEntry__isset __isset; void __set_type(const TTypeId::type val) { type = val; } + void __set_typeQualifiers(const TTypeQualifiers& val) { + typeQualifiers = val; + __isset.typeQualifiers = true; + } + bool operator == (const TPrimitiveTypeEntry & rhs) const { if (!(type == rhs.type)) return false; + if (__isset.typeQualifiers != rhs.__isset.typeQualifiers) + return false; + else if (__isset.typeQualifiers && !(typeQualifiers == rhs.typeQualifiers)) + return false; return true; } bool operator != (const TPrimitiveTypeEntry &rhs) const { @@ -408,8 +519,8 @@ typedef struct _TTypeEntry__isset { class TTypeEntry { public: - static const char* ascii_fingerprint; // = "0C3ACE4054603E2D37B8BFEBA79F4159"; - static const uint8_t binary_fingerprint[16]; // = {0x0C,0x3A,0xCE,0x40,0x54,0x60,0x3E,0x2D,0x37,0xB8,0xBF,0xEB,0xA7,0x9F,0x41,0x59}; + static const char* ascii_fingerprint; // = "2FE56D9097E325DAA7E933738C6D325F"; + static const uint8_t binary_fingerprint[16]; // = {0x2F,0xE5,0x6D,0x90,0x97,0xE3,0x25,0xDA,0xA7,0xE9,0x33,0x73,0x8C,0x6D,0x32,0x5F}; TTypeEntry() { } @@ -482,8 +593,8 @@ void swap(TTypeEntry &a, TTypeEntry &b); class TTypeDesc { public: - static const char* ascii_fingerprint; // = "60CA5B8BACFCD38D1D3EC1F0E3F1C36A"; - static const uint8_t binary_fingerprint[16]; // = {0x60,0xCA,0x5B,0x8B,0xAC,0xFC,0xD3,0x8D,0x1D,0x3E,0xC1,0xF0,0xE3,0xF1,0xC3,0x6A}; + static const char* ascii_fingerprint; // = "90B3C5A0B73419A84E85E0E48C452AA5"; + static const uint8_t binary_fingerprint[16]; // = {0x90,0xB3,0xC5,0xA0,0xB7,0x34,0x19,0xA8,0x4E,0x85,0xE0,0xE4,0x8C,0x45,0x2A,0xA5}; TTypeDesc() { } @@ -523,8 +634,8 @@ typedef struct _TColumnDesc__isset { class TColumnDesc { public: - static const char* ascii_fingerprint; // = "0DF9A37B81B1EE73D35A0AC01F33A48D"; - static const uint8_t binary_fingerprint[16]; // = {0x0D,0xF9,0xA3,0x7B,0x81,0xB1,0xEE,0x73,0xD3,0x5A,0x0A,0xC0,0x1F,0x33,0xA4,0x8D}; + static const char* ascii_fingerprint; // = "EABED9009D5FCABFCA65612069F2A849"; + static const uint8_t binary_fingerprint[16]; // = {0xEA,0xBE,0xD9,0x00,0x9D,0x5F,0xCA,0xBF,0xCA,0x65,0x61,0x20,0x69,0xF2,0xA8,0x49}; TColumnDesc() : columnName(), position(0), comment() { } @@ -586,8 +697,8 @@ void swap(TColumnDesc &a, TColumnDesc &b); class TTableSchema { public: - static const char* ascii_fingerprint; // = "E67E789F1EF836E4B9FC922C788AFDC8"; - static const uint8_t binary_fingerprint[16]; // = {0xE6,0x7E,0x78,0x9F,0x1E,0xF8,0x36,0xE4,0xB9,0xFC,0x92,0x2C,0x78,0x8A,0xFD,0xC8}; + static const char* ascii_fingerprint; // = "7A1811E49313E5977107FC667B20E39D"; + static const uint8_t binary_fingerprint[16]; // = {0x7A,0x18,0x11,0xE4,0x93,0x13,0xE5,0x97,0x71,0x07,0xFC,0x66,0x7B,0x20,0xE3,0x9D}; TTableSchema() { } @@ -3034,8 +3145,8 @@ typedef struct _TGetResultSetMetadataResp__isset { class TGetResultSetMetadataResp { public: - static const char* ascii_fingerprint; // = "8778316D0AFC17584F192162BFF2AEDE"; - static const uint8_t binary_fingerprint[16]; // = {0x87,0x78,0x31,0x6D,0x0A,0xFC,0x17,0x58,0x4F,0x19,0x21,0x62,0xBF,0xF2,0xAE,0xDE}; + static const char* ascii_fingerprint; // = "42CD49B7F49CC1B6D4E6F5FA2D7BE812"; + static const uint8_t binary_fingerprint[16]; // = {0x42,0xCD,0x49,0xB7,0xF4,0x9C,0xC1,0xB6,0xD4,0xE6,0xF5,0xFA,0x2D,0x7B,0xE8,0x12}; TGetResultSetMetadataResp() { } diff --git service/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/service/ThriftHive.java service/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/service/ThriftHive.java index 745e6cc..1c44789 100644 --- service/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/service/ThriftHive.java +++ service/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/service/ThriftHive.java @@ -3023,7 +3023,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, fetchN_result struc struct.success = new ArrayList(_list0.size); for (int _i1 = 0; _i1 < _list0.size; ++_i1) { - String _elem2; // required + String _elem2; // optional _elem2 = iprot.readString(); struct.success.add(_elem2); } @@ -3122,7 +3122,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, fetchN_result struct struct.success = new ArrayList(_list5.size); for (int _i6 = 0; _i6 < _list5.size; ++_i6) { - String _elem7; // required + String _elem7; // optional _elem7 = iprot.readString(); struct.success.add(_elem7); } @@ -3785,7 +3785,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, fetchAll_result str struct.success = new ArrayList(_list8.size); for (int _i9 = 0; _i9 < _list8.size; ++_i9) { - String _elem10; // required + String _elem10; // optional _elem10 = iprot.readString(); struct.success.add(_elem10); } @@ -3884,7 +3884,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, fetchAll_result stru struct.success = new ArrayList(_list13.size); for (int _i14 = 0; _i14 < _list13.size; ++_i14) { - String _elem15; // required + String _elem15; // optional _elem15 = iprot.readString(); struct.success.add(_elem15); } diff --git service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TCLIServiceConstants.java service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TCLIServiceConstants.java index 83b73fa..4748e79 100644 --- service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TCLIServiceConstants.java +++ service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TCLIServiceConstants.java @@ -48,6 +48,7 @@ PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.DECIMAL_TYPE); PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.NULL_TYPE); PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.DATE_TYPE); + PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.VARCHAR_TYPE); } public static final Set COMPLEX_TYPES = new HashSet(); @@ -84,6 +85,9 @@ TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.DECIMAL_TYPE, "DECIMAL"); TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.NULL_TYPE, "NULL"); TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.DATE_TYPE, "DATE"); + TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.VARCHAR_TYPE, "VARCHAR"); } + public static final String CHARACTER_MAXIMUM_LENGTH = "characterMaximumLength"; + } diff --git service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TColumn.java service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TColumn.java index 2b8af2c..497cc01 100644 --- service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TColumn.java +++ service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TColumn.java @@ -255,14 +255,14 @@ protected Object standardSchemeReadValue(org.apache.thrift.protocol.TProtocol ip if (field.type == BOOL_COLUMN_FIELD_DESC.type) { List boolColumn; { - org.apache.thrift.protocol.TList _list36 = iprot.readListBegin(); - boolColumn = new ArrayList(_list36.size); - for (int _i37 = 0; _i37 < _list36.size; ++_i37) + org.apache.thrift.protocol.TList _list46 = iprot.readListBegin(); + boolColumn = new ArrayList(_list46.size); + for (int _i47 = 0; _i47 < _list46.size; ++_i47) { - TBoolValue _elem38; // required - _elem38 = new TBoolValue(); - _elem38.read(iprot); - boolColumn.add(_elem38); + TBoolValue _elem48; // optional + _elem48 = new TBoolValue(); + _elem48.read(iprot); + boolColumn.add(_elem48); } iprot.readListEnd(); } @@ -275,14 +275,14 @@ protected Object standardSchemeReadValue(org.apache.thrift.protocol.TProtocol ip if (field.type == BYTE_COLUMN_FIELD_DESC.type) { List byteColumn; { - org.apache.thrift.protocol.TList _list39 = iprot.readListBegin(); - byteColumn = new ArrayList(_list39.size); - for (int _i40 = 0; _i40 < _list39.size; ++_i40) + org.apache.thrift.protocol.TList _list49 = iprot.readListBegin(); + byteColumn = new ArrayList(_list49.size); + for (int _i50 = 0; _i50 < _list49.size; ++_i50) { - TByteValue _elem41; // required - _elem41 = new TByteValue(); - _elem41.read(iprot); - byteColumn.add(_elem41); + TByteValue _elem51; // optional + _elem51 = new TByteValue(); + _elem51.read(iprot); + byteColumn.add(_elem51); } iprot.readListEnd(); } @@ -295,14 +295,14 @@ protected Object standardSchemeReadValue(org.apache.thrift.protocol.TProtocol ip if (field.type == I16_COLUMN_FIELD_DESC.type) { List i16Column; { - org.apache.thrift.protocol.TList _list42 = iprot.readListBegin(); - i16Column = new ArrayList(_list42.size); - for (int _i43 = 0; _i43 < _list42.size; ++_i43) + org.apache.thrift.protocol.TList _list52 = iprot.readListBegin(); + i16Column = new ArrayList(_list52.size); + for (int _i53 = 0; _i53 < _list52.size; ++_i53) { - TI16Value _elem44; // required - _elem44 = new TI16Value(); - _elem44.read(iprot); - i16Column.add(_elem44); + TI16Value _elem54; // optional + _elem54 = new TI16Value(); + _elem54.read(iprot); + i16Column.add(_elem54); } iprot.readListEnd(); } @@ -315,14 +315,14 @@ protected Object standardSchemeReadValue(org.apache.thrift.protocol.TProtocol ip if (field.type == I32_COLUMN_FIELD_DESC.type) { List i32Column; { - org.apache.thrift.protocol.TList _list45 = iprot.readListBegin(); - i32Column = new ArrayList(_list45.size); - for (int _i46 = 0; _i46 < _list45.size; ++_i46) + org.apache.thrift.protocol.TList _list55 = iprot.readListBegin(); + i32Column = new ArrayList(_list55.size); + for (int _i56 = 0; _i56 < _list55.size; ++_i56) { - TI32Value _elem47; // required - _elem47 = new TI32Value(); - _elem47.read(iprot); - i32Column.add(_elem47); + TI32Value _elem57; // optional + _elem57 = new TI32Value(); + _elem57.read(iprot); + i32Column.add(_elem57); } iprot.readListEnd(); } @@ -335,14 +335,14 @@ protected Object standardSchemeReadValue(org.apache.thrift.protocol.TProtocol ip if (field.type == I64_COLUMN_FIELD_DESC.type) { List i64Column; { - org.apache.thrift.protocol.TList _list48 = iprot.readListBegin(); - i64Column = new ArrayList(_list48.size); - for (int _i49 = 0; _i49 < _list48.size; ++_i49) + org.apache.thrift.protocol.TList _list58 = iprot.readListBegin(); + i64Column = new ArrayList(_list58.size); + for (int _i59 = 0; _i59 < _list58.size; ++_i59) { - TI64Value _elem50; // required - _elem50 = new TI64Value(); - _elem50.read(iprot); - i64Column.add(_elem50); + TI64Value _elem60; // optional + _elem60 = new TI64Value(); + _elem60.read(iprot); + i64Column.add(_elem60); } iprot.readListEnd(); } @@ -355,14 +355,14 @@ protected Object standardSchemeReadValue(org.apache.thrift.protocol.TProtocol ip if (field.type == DOUBLE_COLUMN_FIELD_DESC.type) { List doubleColumn; { - org.apache.thrift.protocol.TList _list51 = iprot.readListBegin(); - doubleColumn = new ArrayList(_list51.size); - for (int _i52 = 0; _i52 < _list51.size; ++_i52) + org.apache.thrift.protocol.TList _list61 = iprot.readListBegin(); + doubleColumn = new ArrayList(_list61.size); + for (int _i62 = 0; _i62 < _list61.size; ++_i62) { - TDoubleValue _elem53; // required - _elem53 = new TDoubleValue(); - _elem53.read(iprot); - doubleColumn.add(_elem53); + TDoubleValue _elem63; // optional + _elem63 = new TDoubleValue(); + _elem63.read(iprot); + doubleColumn.add(_elem63); } iprot.readListEnd(); } @@ -375,14 +375,14 @@ protected Object standardSchemeReadValue(org.apache.thrift.protocol.TProtocol ip if (field.type == STRING_COLUMN_FIELD_DESC.type) { List stringColumn; { - org.apache.thrift.protocol.TList _list54 = iprot.readListBegin(); - stringColumn = new ArrayList(_list54.size); - for (int _i55 = 0; _i55 < _list54.size; ++_i55) + org.apache.thrift.protocol.TList _list64 = iprot.readListBegin(); + stringColumn = new ArrayList(_list64.size); + for (int _i65 = 0; _i65 < _list64.size; ++_i65) { - TStringValue _elem56; // required - _elem56 = new TStringValue(); - _elem56.read(iprot); - stringColumn.add(_elem56); + TStringValue _elem66; // optional + _elem66 = new TStringValue(); + _elem66.read(iprot); + stringColumn.add(_elem66); } iprot.readListEnd(); } @@ -406,9 +406,9 @@ protected void standardSchemeWriteValue(org.apache.thrift.protocol.TProtocol opr List boolColumn = (List)value_; { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, boolColumn.size())); - for (TBoolValue _iter57 : boolColumn) + for (TBoolValue _iter67 : boolColumn) { - _iter57.write(oprot); + _iter67.write(oprot); } oprot.writeListEnd(); } @@ -417,9 +417,9 @@ protected void standardSchemeWriteValue(org.apache.thrift.protocol.TProtocol opr List byteColumn = (List)value_; { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, byteColumn.size())); - for (TByteValue _iter58 : byteColumn) + for (TByteValue _iter68 : byteColumn) { - _iter58.write(oprot); + _iter68.write(oprot); } oprot.writeListEnd(); } @@ -428,9 +428,9 @@ protected void standardSchemeWriteValue(org.apache.thrift.protocol.TProtocol opr List i16Column = (List)value_; { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, i16Column.size())); - for (TI16Value _iter59 : i16Column) + for (TI16Value _iter69 : i16Column) { - _iter59.write(oprot); + _iter69.write(oprot); } oprot.writeListEnd(); } @@ -439,9 +439,9 @@ protected void standardSchemeWriteValue(org.apache.thrift.protocol.TProtocol opr List i32Column = (List)value_; { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, i32Column.size())); - for (TI32Value _iter60 : i32Column) + for (TI32Value _iter70 : i32Column) { - _iter60.write(oprot); + _iter70.write(oprot); } oprot.writeListEnd(); } @@ -450,9 +450,9 @@ protected void standardSchemeWriteValue(org.apache.thrift.protocol.TProtocol opr List i64Column = (List)value_; { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, i64Column.size())); - for (TI64Value _iter61 : i64Column) + for (TI64Value _iter71 : i64Column) { - _iter61.write(oprot); + _iter71.write(oprot); } oprot.writeListEnd(); } @@ -461,9 +461,9 @@ protected void standardSchemeWriteValue(org.apache.thrift.protocol.TProtocol opr List doubleColumn = (List)value_; { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, doubleColumn.size())); - for (TDoubleValue _iter62 : doubleColumn) + for (TDoubleValue _iter72 : doubleColumn) { - _iter62.write(oprot); + _iter72.write(oprot); } oprot.writeListEnd(); } @@ -472,9 +472,9 @@ protected void standardSchemeWriteValue(org.apache.thrift.protocol.TProtocol opr List stringColumn = (List)value_; { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, stringColumn.size())); - for (TStringValue _iter63 : stringColumn) + for (TStringValue _iter73 : stringColumn) { - _iter63.write(oprot); + _iter73.write(oprot); } oprot.writeListEnd(); } @@ -492,14 +492,14 @@ protected Object tupleSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot case BOOL_COLUMN: List boolColumn; { - org.apache.thrift.protocol.TList _list64 = iprot.readListBegin(); - boolColumn = new ArrayList(_list64.size); - for (int _i65 = 0; _i65 < _list64.size; ++_i65) + org.apache.thrift.protocol.TList _list74 = iprot.readListBegin(); + boolColumn = new ArrayList(_list74.size); + for (int _i75 = 0; _i75 < _list74.size; ++_i75) { - TBoolValue _elem66; // required - _elem66 = new TBoolValue(); - _elem66.read(iprot); - boolColumn.add(_elem66); + TBoolValue _elem76; // optional + _elem76 = new TBoolValue(); + _elem76.read(iprot); + boolColumn.add(_elem76); } iprot.readListEnd(); } @@ -507,14 +507,14 @@ protected Object tupleSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot case BYTE_COLUMN: List byteColumn; { - org.apache.thrift.protocol.TList _list67 = iprot.readListBegin(); - byteColumn = new ArrayList(_list67.size); - for (int _i68 = 0; _i68 < _list67.size; ++_i68) + org.apache.thrift.protocol.TList _list77 = iprot.readListBegin(); + byteColumn = new ArrayList(_list77.size); + for (int _i78 = 0; _i78 < _list77.size; ++_i78) { - TByteValue _elem69; // required - _elem69 = new TByteValue(); - _elem69.read(iprot); - byteColumn.add(_elem69); + TByteValue _elem79; // optional + _elem79 = new TByteValue(); + _elem79.read(iprot); + byteColumn.add(_elem79); } iprot.readListEnd(); } @@ -522,14 +522,14 @@ protected Object tupleSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot case I16_COLUMN: List i16Column; { - org.apache.thrift.protocol.TList _list70 = iprot.readListBegin(); - i16Column = new ArrayList(_list70.size); - for (int _i71 = 0; _i71 < _list70.size; ++_i71) + org.apache.thrift.protocol.TList _list80 = iprot.readListBegin(); + i16Column = new ArrayList(_list80.size); + for (int _i81 = 0; _i81 < _list80.size; ++_i81) { - TI16Value _elem72; // required - _elem72 = new TI16Value(); - _elem72.read(iprot); - i16Column.add(_elem72); + TI16Value _elem82; // optional + _elem82 = new TI16Value(); + _elem82.read(iprot); + i16Column.add(_elem82); } iprot.readListEnd(); } @@ -537,14 +537,14 @@ protected Object tupleSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot case I32_COLUMN: List i32Column; { - org.apache.thrift.protocol.TList _list73 = iprot.readListBegin(); - i32Column = new ArrayList(_list73.size); - for (int _i74 = 0; _i74 < _list73.size; ++_i74) + org.apache.thrift.protocol.TList _list83 = iprot.readListBegin(); + i32Column = new ArrayList(_list83.size); + for (int _i84 = 0; _i84 < _list83.size; ++_i84) { - TI32Value _elem75; // required - _elem75 = new TI32Value(); - _elem75.read(iprot); - i32Column.add(_elem75); + TI32Value _elem85; // optional + _elem85 = new TI32Value(); + _elem85.read(iprot); + i32Column.add(_elem85); } iprot.readListEnd(); } @@ -552,14 +552,14 @@ protected Object tupleSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot case I64_COLUMN: List i64Column; { - org.apache.thrift.protocol.TList _list76 = iprot.readListBegin(); - i64Column = new ArrayList(_list76.size); - for (int _i77 = 0; _i77 < _list76.size; ++_i77) + org.apache.thrift.protocol.TList _list86 = iprot.readListBegin(); + i64Column = new ArrayList(_list86.size); + for (int _i87 = 0; _i87 < _list86.size; ++_i87) { - TI64Value _elem78; // required - _elem78 = new TI64Value(); - _elem78.read(iprot); - i64Column.add(_elem78); + TI64Value _elem88; // optional + _elem88 = new TI64Value(); + _elem88.read(iprot); + i64Column.add(_elem88); } iprot.readListEnd(); } @@ -567,14 +567,14 @@ protected Object tupleSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot case DOUBLE_COLUMN: List doubleColumn; { - org.apache.thrift.protocol.TList _list79 = iprot.readListBegin(); - doubleColumn = new ArrayList(_list79.size); - for (int _i80 = 0; _i80 < _list79.size; ++_i80) + org.apache.thrift.protocol.TList _list89 = iprot.readListBegin(); + doubleColumn = new ArrayList(_list89.size); + for (int _i90 = 0; _i90 < _list89.size; ++_i90) { - TDoubleValue _elem81; // required - _elem81 = new TDoubleValue(); - _elem81.read(iprot); - doubleColumn.add(_elem81); + TDoubleValue _elem91; // optional + _elem91 = new TDoubleValue(); + _elem91.read(iprot); + doubleColumn.add(_elem91); } iprot.readListEnd(); } @@ -582,14 +582,14 @@ protected Object tupleSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot case STRING_COLUMN: List stringColumn; { - org.apache.thrift.protocol.TList _list82 = iprot.readListBegin(); - stringColumn = new ArrayList(_list82.size); - for (int _i83 = 0; _i83 < _list82.size; ++_i83) + org.apache.thrift.protocol.TList _list92 = iprot.readListBegin(); + stringColumn = new ArrayList(_list92.size); + for (int _i93 = 0; _i93 < _list92.size; ++_i93) { - TStringValue _elem84; // required - _elem84 = new TStringValue(); - _elem84.read(iprot); - stringColumn.add(_elem84); + TStringValue _elem94; // optional + _elem94 = new TStringValue(); + _elem94.read(iprot); + stringColumn.add(_elem94); } iprot.readListEnd(); } @@ -609,9 +609,9 @@ protected void tupleSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) List boolColumn = (List)value_; { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, boolColumn.size())); - for (TBoolValue _iter85 : boolColumn) + for (TBoolValue _iter95 : boolColumn) { - _iter85.write(oprot); + _iter95.write(oprot); } oprot.writeListEnd(); } @@ -620,9 +620,9 @@ protected void tupleSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) List byteColumn = (List)value_; { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, byteColumn.size())); - for (TByteValue _iter86 : byteColumn) + for (TByteValue _iter96 : byteColumn) { - _iter86.write(oprot); + _iter96.write(oprot); } oprot.writeListEnd(); } @@ -631,9 +631,9 @@ protected void tupleSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) List i16Column = (List)value_; { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, i16Column.size())); - for (TI16Value _iter87 : i16Column) + for (TI16Value _iter97 : i16Column) { - _iter87.write(oprot); + _iter97.write(oprot); } oprot.writeListEnd(); } @@ -642,9 +642,9 @@ protected void tupleSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) List i32Column = (List)value_; { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, i32Column.size())); - for (TI32Value _iter88 : i32Column) + for (TI32Value _iter98 : i32Column) { - _iter88.write(oprot); + _iter98.write(oprot); } oprot.writeListEnd(); } @@ -653,9 +653,9 @@ protected void tupleSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) List i64Column = (List)value_; { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, i64Column.size())); - for (TI64Value _iter89 : i64Column) + for (TI64Value _iter99 : i64Column) { - _iter89.write(oprot); + _iter99.write(oprot); } oprot.writeListEnd(); } @@ -664,9 +664,9 @@ protected void tupleSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) List doubleColumn = (List)value_; { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, doubleColumn.size())); - for (TDoubleValue _iter90 : doubleColumn) + for (TDoubleValue _iter100 : doubleColumn) { - _iter90.write(oprot); + _iter100.write(oprot); } oprot.writeListEnd(); } @@ -675,9 +675,9 @@ protected void tupleSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) List stringColumn = (List)value_; { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, stringColumn.size())); - for (TStringValue _iter91 : stringColumn) + for (TStringValue _iter101 : stringColumn) { - _iter91.write(oprot); + _iter101.write(oprot); } oprot.writeListEnd(); } diff --git service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TExecuteStatementReq.java service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TExecuteStatementReq.java index 24568a1..ca6283d 100644 --- service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TExecuteStatementReq.java +++ service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TExecuteStatementReq.java @@ -542,15 +542,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TExecuteStatementRe case 3: // CONF_OVERLAY if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { { - org.apache.thrift.protocol.TMap _map144 = iprot.readMapBegin(); - struct.confOverlay = new HashMap(2*_map144.size); - for (int _i145 = 0; _i145 < _map144.size; ++_i145) + org.apache.thrift.protocol.TMap _map154 = iprot.readMapBegin(); + struct.confOverlay = new HashMap(2*_map154.size); + for (int _i155 = 0; _i155 < _map154.size; ++_i155) { - String _key146; // required - String _val147; // required - _key146 = iprot.readString(); - _val147 = iprot.readString(); - struct.confOverlay.put(_key146, _val147); + String _key156; // required + String _val157; // required + _key156 = iprot.readString(); + _val157 = iprot.readString(); + struct.confOverlay.put(_key156, _val157); } iprot.readMapEnd(); } @@ -587,10 +587,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TExecuteStatementR oprot.writeFieldBegin(CONF_OVERLAY_FIELD_DESC); { oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.confOverlay.size())); - for (Map.Entry _iter148 : struct.confOverlay.entrySet()) + for (Map.Entry _iter158 : struct.confOverlay.entrySet()) { - oprot.writeString(_iter148.getKey()); - oprot.writeString(_iter148.getValue()); + oprot.writeString(_iter158.getKey()); + oprot.writeString(_iter158.getValue()); } oprot.writeMapEnd(); } @@ -624,10 +624,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TExecuteStatementRe if (struct.isSetConfOverlay()) { { oprot.writeI32(struct.confOverlay.size()); - for (Map.Entry _iter149 : struct.confOverlay.entrySet()) + for (Map.Entry _iter159 : struct.confOverlay.entrySet()) { - oprot.writeString(_iter149.getKey()); - oprot.writeString(_iter149.getValue()); + oprot.writeString(_iter159.getKey()); + oprot.writeString(_iter159.getValue()); } } } @@ -644,15 +644,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TExecuteStatementReq BitSet incoming = iprot.readBitSet(1); if (incoming.get(0)) { { - org.apache.thrift.protocol.TMap _map150 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); - struct.confOverlay = new HashMap(2*_map150.size); - for (int _i151 = 0; _i151 < _map150.size; ++_i151) + org.apache.thrift.protocol.TMap _map160 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.confOverlay = new HashMap(2*_map160.size); + for (int _i161 = 0; _i161 < _map160.size; ++_i161) { - String _key152; // required - String _val153; // required - _key152 = iprot.readString(); - _val153 = iprot.readString(); - struct.confOverlay.put(_key152, _val153); + String _key162; // required + String _val163; // required + _key162 = iprot.readString(); + _val163 = iprot.readString(); + struct.confOverlay.put(_key162, _val163); } } struct.setConfOverlayIsSet(true); diff --git service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TGetTablesReq.java service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TGetTablesReq.java index a59aa49..1cb5147 100644 --- service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TGetTablesReq.java +++ service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TGetTablesReq.java @@ -711,13 +711,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TGetTablesReq struc case 5: // TABLE_TYPES if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { { - org.apache.thrift.protocol.TList _list154 = iprot.readListBegin(); - struct.tableTypes = new ArrayList(_list154.size); - for (int _i155 = 0; _i155 < _list154.size; ++_i155) + org.apache.thrift.protocol.TList _list164 = iprot.readListBegin(); + struct.tableTypes = new ArrayList(_list164.size); + for (int _i165 = 0; _i165 < _list164.size; ++_i165) { - String _elem156; // required - _elem156 = iprot.readString(); - struct.tableTypes.add(_elem156); + String _elem166; // optional + _elem166 = iprot.readString(); + struct.tableTypes.add(_elem166); } iprot.readListEnd(); } @@ -770,9 +770,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TGetTablesReq stru oprot.writeFieldBegin(TABLE_TYPES_FIELD_DESC); { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.tableTypes.size())); - for (String _iter157 : struct.tableTypes) + for (String _iter167 : struct.tableTypes) { - oprot.writeString(_iter157); + oprot.writeString(_iter167); } oprot.writeListEnd(); } @@ -823,9 +823,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TGetTablesReq struc if (struct.isSetTableTypes()) { { oprot.writeI32(struct.tableTypes.size()); - for (String _iter158 : struct.tableTypes) + for (String _iter168 : struct.tableTypes) { - oprot.writeString(_iter158); + oprot.writeString(_iter168); } } } @@ -852,13 +852,13 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TGetTablesReq struct } if (incoming.get(3)) { { - org.apache.thrift.protocol.TList _list159 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); - struct.tableTypes = new ArrayList(_list159.size); - for (int _i160 = 0; _i160 < _list159.size; ++_i160) + org.apache.thrift.protocol.TList _list169 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.tableTypes = new ArrayList(_list169.size); + for (int _i170 = 0; _i170 < _list169.size; ++_i170) { - String _elem161; // required - _elem161 = iprot.readString(); - struct.tableTypes.add(_elem161); + String _elem171; // optional + _elem171 = iprot.readString(); + struct.tableTypes.add(_elem171); } } struct.setTableTypesIsSet(true); diff --git service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TOpenSessionReq.java service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TOpenSessionReq.java index d026adf..b537260 100644 --- service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TOpenSessionReq.java +++ service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TOpenSessionReq.java @@ -638,15 +638,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TOpenSessionReq str case 4: // CONFIGURATION if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { { - org.apache.thrift.protocol.TMap _map124 = iprot.readMapBegin(); - struct.configuration = new HashMap(2*_map124.size); - for (int _i125 = 0; _i125 < _map124.size; ++_i125) + org.apache.thrift.protocol.TMap _map134 = iprot.readMapBegin(); + struct.configuration = new HashMap(2*_map134.size); + for (int _i135 = 0; _i135 < _map134.size; ++_i135) { - String _key126; // required - String _val127; // required - _key126 = iprot.readString(); - _val127 = iprot.readString(); - struct.configuration.put(_key126, _val127); + String _key136; // required + String _val137; // required + _key136 = iprot.readString(); + _val137 = iprot.readString(); + struct.configuration.put(_key136, _val137); } iprot.readMapEnd(); } @@ -692,10 +692,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TOpenSessionReq st oprot.writeFieldBegin(CONFIGURATION_FIELD_DESC); { oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.configuration.size())); - for (Map.Entry _iter128 : struct.configuration.entrySet()) + for (Map.Entry _iter138 : struct.configuration.entrySet()) { - oprot.writeString(_iter128.getKey()); - oprot.writeString(_iter128.getValue()); + oprot.writeString(_iter138.getKey()); + oprot.writeString(_iter138.getValue()); } oprot.writeMapEnd(); } @@ -740,10 +740,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TOpenSessionReq str if (struct.isSetConfiguration()) { { oprot.writeI32(struct.configuration.size()); - for (Map.Entry _iter129 : struct.configuration.entrySet()) + for (Map.Entry _iter139 : struct.configuration.entrySet()) { - oprot.writeString(_iter129.getKey()); - oprot.writeString(_iter129.getValue()); + oprot.writeString(_iter139.getKey()); + oprot.writeString(_iter139.getValue()); } } } @@ -765,15 +765,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TOpenSessionReq stru } if (incoming.get(2)) { { - org.apache.thrift.protocol.TMap _map130 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); - struct.configuration = new HashMap(2*_map130.size); - for (int _i131 = 0; _i131 < _map130.size; ++_i131) + org.apache.thrift.protocol.TMap _map140 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.configuration = new HashMap(2*_map140.size); + for (int _i141 = 0; _i141 < _map140.size; ++_i141) { - String _key132; // required - String _val133; // required - _key132 = iprot.readString(); - _val133 = iprot.readString(); - struct.configuration.put(_key132, _val133); + String _key142; // required + String _val143; // required + _key142 = iprot.readString(); + _val143 = iprot.readString(); + struct.configuration.put(_key142, _val143); } } struct.setConfigurationIsSet(true); diff --git service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TOpenSessionResp.java service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TOpenSessionResp.java index e70d63a..350d23e 100644 --- service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TOpenSessionResp.java +++ service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TOpenSessionResp.java @@ -650,15 +650,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TOpenSessionResp st case 4: // CONFIGURATION if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { { - org.apache.thrift.protocol.TMap _map134 = iprot.readMapBegin(); - struct.configuration = new HashMap(2*_map134.size); - for (int _i135 = 0; _i135 < _map134.size; ++_i135) + org.apache.thrift.protocol.TMap _map144 = iprot.readMapBegin(); + struct.configuration = new HashMap(2*_map144.size); + for (int _i145 = 0; _i145 < _map144.size; ++_i145) { - String _key136; // required - String _val137; // required - _key136 = iprot.readString(); - _val137 = iprot.readString(); - struct.configuration.put(_key136, _val137); + String _key146; // required + String _val147; // required + _key146 = iprot.readString(); + _val147 = iprot.readString(); + struct.configuration.put(_key146, _val147); } iprot.readMapEnd(); } @@ -702,10 +702,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TOpenSessionResp s oprot.writeFieldBegin(CONFIGURATION_FIELD_DESC); { oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.configuration.size())); - for (Map.Entry _iter138 : struct.configuration.entrySet()) + for (Map.Entry _iter148 : struct.configuration.entrySet()) { - oprot.writeString(_iter138.getKey()); - oprot.writeString(_iter138.getValue()); + oprot.writeString(_iter148.getKey()); + oprot.writeString(_iter148.getValue()); } oprot.writeMapEnd(); } @@ -745,10 +745,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TOpenSessionResp st if (struct.isSetConfiguration()) { { oprot.writeI32(struct.configuration.size()); - for (Map.Entry _iter139 : struct.configuration.entrySet()) + for (Map.Entry _iter149 : struct.configuration.entrySet()) { - oprot.writeString(_iter139.getKey()); - oprot.writeString(_iter139.getValue()); + oprot.writeString(_iter149.getKey()); + oprot.writeString(_iter149.getValue()); } } } @@ -770,15 +770,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TOpenSessionResp str } if (incoming.get(1)) { { - org.apache.thrift.protocol.TMap _map140 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); - struct.configuration = new HashMap(2*_map140.size); - for (int _i141 = 0; _i141 < _map140.size; ++_i141) + org.apache.thrift.protocol.TMap _map150 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.configuration = new HashMap(2*_map150.size); + for (int _i151 = 0; _i151 < _map150.size; ++_i151) { - String _key142; // required - String _val143; // required - _key142 = iprot.readString(); - _val143 = iprot.readString(); - struct.configuration.put(_key142, _val143); + String _key152; // required + String _val153; // required + _key152 = iprot.readString(); + _val153 = iprot.readString(); + struct.configuration.put(_key152, _val153); } } struct.setConfigurationIsSet(true); diff --git service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TPrimitiveTypeEntry.java service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TPrimitiveTypeEntry.java index c0e2e28..9d2abf2 100644 --- service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TPrimitiveTypeEntry.java +++ service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TPrimitiveTypeEntry.java @@ -35,6 +35,7 @@ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TPrimitiveTypeEntry"); private static final org.apache.thrift.protocol.TField TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("type", org.apache.thrift.protocol.TType.I32, (short)1); + private static final org.apache.thrift.protocol.TField TYPE_QUALIFIERS_FIELD_DESC = new org.apache.thrift.protocol.TField("typeQualifiers", org.apache.thrift.protocol.TType.STRUCT, (short)2); private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { @@ -43,6 +44,7 @@ } private TTypeId type; // required + private TTypeQualifiers typeQualifiers; // optional /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { @@ -50,7 +52,8 @@ * * @see TTypeId */ - TYPE((short)1, "type"); + TYPE((short)1, "type"), + TYPE_QUALIFIERS((short)2, "typeQualifiers"); private static final Map byName = new HashMap(); @@ -67,6 +70,8 @@ public static _Fields findByThriftId(int fieldId) { switch(fieldId) { case 1: // TYPE return TYPE; + case 2: // TYPE_QUALIFIERS + return TYPE_QUALIFIERS; default: return null; } @@ -107,11 +112,14 @@ public String getFieldName() { } // isset id assignments + private _Fields optionals[] = {_Fields.TYPE_QUALIFIERS}; public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); tmpMap.put(_Fields.TYPE, new org.apache.thrift.meta_data.FieldMetaData("type", org.apache.thrift.TFieldRequirementType.REQUIRED, new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TTypeId.class))); + tmpMap.put(_Fields.TYPE_QUALIFIERS, new org.apache.thrift.meta_data.FieldMetaData("typeQualifiers", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TTypeQualifiers.class))); metaDataMap = Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TPrimitiveTypeEntry.class, metaDataMap); } @@ -133,6 +141,9 @@ public TPrimitiveTypeEntry(TPrimitiveTypeEntry other) { if (other.isSetType()) { this.type = other.type; } + if (other.isSetTypeQualifiers()) { + this.typeQualifiers = new TTypeQualifiers(other.typeQualifiers); + } } public TPrimitiveTypeEntry deepCopy() { @@ -142,6 +153,7 @@ public TPrimitiveTypeEntry deepCopy() { @Override public void clear() { this.type = null; + this.typeQualifiers = null; } /** @@ -175,6 +187,29 @@ public void setTypeIsSet(boolean value) { } } + public TTypeQualifiers getTypeQualifiers() { + return this.typeQualifiers; + } + + public void setTypeQualifiers(TTypeQualifiers typeQualifiers) { + this.typeQualifiers = typeQualifiers; + } + + public void unsetTypeQualifiers() { + this.typeQualifiers = null; + } + + /** Returns true if field typeQualifiers is set (has been assigned a value) and false otherwise */ + public boolean isSetTypeQualifiers() { + return this.typeQualifiers != null; + } + + public void setTypeQualifiersIsSet(boolean value) { + if (!value) { + this.typeQualifiers = null; + } + } + public void setFieldValue(_Fields field, Object value) { switch (field) { case TYPE: @@ -185,6 +220,14 @@ public void setFieldValue(_Fields field, Object value) { } break; + case TYPE_QUALIFIERS: + if (value == null) { + unsetTypeQualifiers(); + } else { + setTypeQualifiers((TTypeQualifiers)value); + } + break; + } } @@ -193,6 +236,9 @@ public Object getFieldValue(_Fields field) { case TYPE: return getType(); + case TYPE_QUALIFIERS: + return getTypeQualifiers(); + } throw new IllegalStateException(); } @@ -206,6 +252,8 @@ public boolean isSet(_Fields field) { switch (field) { case TYPE: return isSetType(); + case TYPE_QUALIFIERS: + return isSetTypeQualifiers(); } throw new IllegalStateException(); } @@ -232,6 +280,15 @@ public boolean equals(TPrimitiveTypeEntry that) { return false; } + boolean this_present_typeQualifiers = true && this.isSetTypeQualifiers(); + boolean that_present_typeQualifiers = true && that.isSetTypeQualifiers(); + if (this_present_typeQualifiers || that_present_typeQualifiers) { + if (!(this_present_typeQualifiers && that_present_typeQualifiers)) + return false; + if (!this.typeQualifiers.equals(that.typeQualifiers)) + return false; + } + return true; } @@ -244,6 +301,11 @@ public int hashCode() { if (present_type) builder.append(type.getValue()); + boolean present_typeQualifiers = true && (isSetTypeQualifiers()); + builder.append(present_typeQualifiers); + if (present_typeQualifiers) + builder.append(typeQualifiers); + return builder.toHashCode(); } @@ -265,6 +327,16 @@ public int compareTo(TPrimitiveTypeEntry other) { return lastComparison; } } + lastComparison = Boolean.valueOf(isSetTypeQualifiers()).compareTo(typedOther.isSetTypeQualifiers()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTypeQualifiers()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.typeQualifiers, typedOther.typeQualifiers); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -292,6 +364,16 @@ public String toString() { sb.append(this.type); } first = false; + if (isSetTypeQualifiers()) { + if (!first) sb.append(", "); + sb.append("typeQualifiers:"); + if (this.typeQualifiers == null) { + sb.append("null"); + } else { + sb.append(this.typeQualifiers); + } + first = false; + } sb.append(")"); return sb.toString(); } @@ -303,6 +385,9 @@ public void validate() throws org.apache.thrift.TException { } // check for sub-struct validity + if (typeQualifiers != null) { + typeQualifiers.validate(); + } } private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { @@ -347,6 +432,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TPrimitiveTypeEntry org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; + case 2: // TYPE_QUALIFIERS + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.typeQualifiers = new TTypeQualifiers(); + struct.typeQualifiers.read(iprot); + struct.setTypeQualifiersIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -365,6 +459,13 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TPrimitiveTypeEntr oprot.writeI32(struct.type.getValue()); oprot.writeFieldEnd(); } + if (struct.typeQualifiers != null) { + if (struct.isSetTypeQualifiers()) { + oprot.writeFieldBegin(TYPE_QUALIFIERS_FIELD_DESC); + struct.typeQualifiers.write(oprot); + oprot.writeFieldEnd(); + } + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -383,6 +484,14 @@ public TPrimitiveTypeEntryTupleScheme getScheme() { public void write(org.apache.thrift.protocol.TProtocol prot, TPrimitiveTypeEntry struct) throws org.apache.thrift.TException { TTupleProtocol oprot = (TTupleProtocol) prot; oprot.writeI32(struct.type.getValue()); + BitSet optionals = new BitSet(); + if (struct.isSetTypeQualifiers()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetTypeQualifiers()) { + struct.typeQualifiers.write(oprot); + } } @Override @@ -390,6 +499,12 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TPrimitiveTypeEntry TTupleProtocol iprot = (TTupleProtocol) prot; struct.type = TTypeId.findByValue(iprot.readI32()); struct.setTypeIsSet(true); + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.typeQualifiers = new TTypeQualifiers(); + struct.typeQualifiers.read(iprot); + struct.setTypeQualifiersIsSet(true); + } } } diff --git service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TRow.java service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TRow.java index 82b2ae9..0b6772c 100644 --- service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TRow.java +++ service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TRow.java @@ -350,14 +350,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TRow struct) throws case 1: // COL_VALS if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { { - org.apache.thrift.protocol.TList _list92 = iprot.readListBegin(); - struct.colVals = new ArrayList(_list92.size); - for (int _i93 = 0; _i93 < _list92.size; ++_i93) + org.apache.thrift.protocol.TList _list102 = iprot.readListBegin(); + struct.colVals = new ArrayList(_list102.size); + for (int _i103 = 0; _i103 < _list102.size; ++_i103) { - TColumnValue _elem94; // required - _elem94 = new TColumnValue(); - _elem94.read(iprot); - struct.colVals.add(_elem94); + TColumnValue _elem104; // optional + _elem104 = new TColumnValue(); + _elem104.read(iprot); + struct.colVals.add(_elem104); } iprot.readListEnd(); } @@ -383,9 +383,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TRow struct) throw oprot.writeFieldBegin(COL_VALS_FIELD_DESC); { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.colVals.size())); - for (TColumnValue _iter95 : struct.colVals) + for (TColumnValue _iter105 : struct.colVals) { - _iter95.write(oprot); + _iter105.write(oprot); } oprot.writeListEnd(); } @@ -410,9 +410,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TRow struct) throws TTupleProtocol oprot = (TTupleProtocol) prot; { oprot.writeI32(struct.colVals.size()); - for (TColumnValue _iter96 : struct.colVals) + for (TColumnValue _iter106 : struct.colVals) { - _iter96.write(oprot); + _iter106.write(oprot); } } } @@ -421,14 +421,14 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TRow struct) throws public void read(org.apache.thrift.protocol.TProtocol prot, TRow struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; { - org.apache.thrift.protocol.TList _list97 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); - struct.colVals = new ArrayList(_list97.size); - for (int _i98 = 0; _i98 < _list97.size; ++_i98) + org.apache.thrift.protocol.TList _list107 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.colVals = new ArrayList(_list107.size); + for (int _i108 = 0; _i108 < _list107.size; ++_i108) { - TColumnValue _elem99; // required - _elem99 = new TColumnValue(); - _elem99.read(iprot); - struct.colVals.add(_elem99); + TColumnValue _elem109; // optional + _elem109 = new TColumnValue(); + _elem109.read(iprot); + struct.colVals.add(_elem109); } } struct.setColValsIsSet(true); diff --git service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TRowSet.java service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TRowSet.java index 2e5c5cc..db2262d 100644 --- service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TRowSet.java +++ service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TRowSet.java @@ -545,14 +545,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TRowSet struct) thr case 2: // ROWS if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { { - org.apache.thrift.protocol.TList _list100 = iprot.readListBegin(); - struct.rows = new ArrayList(_list100.size); - for (int _i101 = 0; _i101 < _list100.size; ++_i101) + org.apache.thrift.protocol.TList _list110 = iprot.readListBegin(); + struct.rows = new ArrayList(_list110.size); + for (int _i111 = 0; _i111 < _list110.size; ++_i111) { - TRow _elem102; // required - _elem102 = new TRow(); - _elem102.read(iprot); - struct.rows.add(_elem102); + TRow _elem112; // optional + _elem112 = new TRow(); + _elem112.read(iprot); + struct.rows.add(_elem112); } iprot.readListEnd(); } @@ -564,14 +564,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TRowSet struct) thr case 3: // COLUMNS if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { { - org.apache.thrift.protocol.TList _list103 = iprot.readListBegin(); - struct.columns = new ArrayList(_list103.size); - for (int _i104 = 0; _i104 < _list103.size; ++_i104) + org.apache.thrift.protocol.TList _list113 = iprot.readListBegin(); + struct.columns = new ArrayList(_list113.size); + for (int _i114 = 0; _i114 < _list113.size; ++_i114) { - TColumn _elem105; // required - _elem105 = new TColumn(); - _elem105.read(iprot); - struct.columns.add(_elem105); + TColumn _elem115; // optional + _elem115 = new TColumn(); + _elem115.read(iprot); + struct.columns.add(_elem115); } iprot.readListEnd(); } @@ -600,9 +600,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TRowSet struct) th oprot.writeFieldBegin(ROWS_FIELD_DESC); { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.rows.size())); - for (TRow _iter106 : struct.rows) + for (TRow _iter116 : struct.rows) { - _iter106.write(oprot); + _iter116.write(oprot); } oprot.writeListEnd(); } @@ -613,9 +613,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TRowSet struct) th oprot.writeFieldBegin(COLUMNS_FIELD_DESC); { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.columns.size())); - for (TColumn _iter107 : struct.columns) + for (TColumn _iter117 : struct.columns) { - _iter107.write(oprot); + _iter117.write(oprot); } oprot.writeListEnd(); } @@ -642,9 +642,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TRowSet struct) thr oprot.writeI64(struct.startRowOffset); { oprot.writeI32(struct.rows.size()); - for (TRow _iter108 : struct.rows) + for (TRow _iter118 : struct.rows) { - _iter108.write(oprot); + _iter118.write(oprot); } } BitSet optionals = new BitSet(); @@ -655,9 +655,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TRowSet struct) thr if (struct.isSetColumns()) { { oprot.writeI32(struct.columns.size()); - for (TColumn _iter109 : struct.columns) + for (TColumn _iter119 : struct.columns) { - _iter109.write(oprot); + _iter119.write(oprot); } } } @@ -669,28 +669,28 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TRowSet struct) thro struct.startRowOffset = iprot.readI64(); struct.setStartRowOffsetIsSet(true); { - org.apache.thrift.protocol.TList _list110 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); - struct.rows = new ArrayList(_list110.size); - for (int _i111 = 0; _i111 < _list110.size; ++_i111) + org.apache.thrift.protocol.TList _list120 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.rows = new ArrayList(_list120.size); + for (int _i121 = 0; _i121 < _list120.size; ++_i121) { - TRow _elem112; // required - _elem112 = new TRow(); - _elem112.read(iprot); - struct.rows.add(_elem112); + TRow _elem122; // optional + _elem122 = new TRow(); + _elem122.read(iprot); + struct.rows.add(_elem122); } } struct.setRowsIsSet(true); BitSet incoming = iprot.readBitSet(1); if (incoming.get(0)) { { - org.apache.thrift.protocol.TList _list113 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); - struct.columns = new ArrayList(_list113.size); - for (int _i114 = 0; _i114 < _list113.size; ++_i114) + org.apache.thrift.protocol.TList _list123 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.columns = new ArrayList(_list123.size); + for (int _i124 = 0; _i124 < _list123.size; ++_i124) { - TColumn _elem115; // required - _elem115 = new TColumn(); - _elem115.read(iprot); - struct.columns.add(_elem115); + TColumn _elem125; // optional + _elem125 = new TColumn(); + _elem125.read(iprot); + struct.columns.add(_elem125); } } struct.setColumnsIsSet(true); diff --git service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TStatus.java service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TStatus.java index 300597e..81c2f16 100644 --- service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TStatus.java +++ service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TStatus.java @@ -694,13 +694,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TStatus struct) thr case 2: // INFO_MESSAGES if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { { - org.apache.thrift.protocol.TList _list116 = iprot.readListBegin(); - struct.infoMessages = new ArrayList(_list116.size); - for (int _i117 = 0; _i117 < _list116.size; ++_i117) + org.apache.thrift.protocol.TList _list126 = iprot.readListBegin(); + struct.infoMessages = new ArrayList(_list126.size); + for (int _i127 = 0; _i127 < _list126.size; ++_i127) { - String _elem118; // required - _elem118 = iprot.readString(); - struct.infoMessages.add(_elem118); + String _elem128; // optional + _elem128 = iprot.readString(); + struct.infoMessages.add(_elem128); } iprot.readListEnd(); } @@ -756,9 +756,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TStatus struct) th oprot.writeFieldBegin(INFO_MESSAGES_FIELD_DESC); { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.infoMessages.size())); - for (String _iter119 : struct.infoMessages) + for (String _iter129 : struct.infoMessages) { - oprot.writeString(_iter119); + oprot.writeString(_iter129); } oprot.writeListEnd(); } @@ -819,9 +819,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TStatus struct) thr if (struct.isSetInfoMessages()) { { oprot.writeI32(struct.infoMessages.size()); - for (String _iter120 : struct.infoMessages) + for (String _iter130 : struct.infoMessages) { - oprot.writeString(_iter120); + oprot.writeString(_iter130); } } } @@ -844,13 +844,13 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TStatus struct) thro BitSet incoming = iprot.readBitSet(4); if (incoming.get(0)) { { - org.apache.thrift.protocol.TList _list121 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); - struct.infoMessages = new ArrayList(_list121.size); - for (int _i122 = 0; _i122 < _list121.size; ++_i122) + org.apache.thrift.protocol.TList _list131 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.infoMessages = new ArrayList(_list131.size); + for (int _i132 = 0; _i132 < _list131.size; ++_i132) { - String _elem123; // required - _elem123 = iprot.readString(); - struct.infoMessages.add(_elem123); + String _elem133; // optional + _elem133 = iprot.readString(); + struct.infoMessages.add(_elem133); } } struct.setInfoMessagesIsSet(true); diff --git service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TStructTypeEntry.java service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TStructTypeEntry.java index d12238f..20f5fb6 100644 --- service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TStructTypeEntry.java +++ service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TStructTypeEntry.java @@ -355,15 +355,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TStructTypeEntry st case 1: // NAME_TO_TYPE_PTR if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { { - org.apache.thrift.protocol.TMap _map0 = iprot.readMapBegin(); - struct.nameToTypePtr = new HashMap(2*_map0.size); - for (int _i1 = 0; _i1 < _map0.size; ++_i1) + org.apache.thrift.protocol.TMap _map10 = iprot.readMapBegin(); + struct.nameToTypePtr = new HashMap(2*_map10.size); + for (int _i11 = 0; _i11 < _map10.size; ++_i11) { - String _key2; // required - int _val3; // required - _key2 = iprot.readString(); - _val3 = iprot.readI32(); - struct.nameToTypePtr.put(_key2, _val3); + String _key12; // required + int _val13; // required + _key12 = iprot.readString(); + _val13 = iprot.readI32(); + struct.nameToTypePtr.put(_key12, _val13); } iprot.readMapEnd(); } @@ -389,10 +389,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TStructTypeEntry s oprot.writeFieldBegin(NAME_TO_TYPE_PTR_FIELD_DESC); { oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.I32, struct.nameToTypePtr.size())); - for (Map.Entry _iter4 : struct.nameToTypePtr.entrySet()) + for (Map.Entry _iter14 : struct.nameToTypePtr.entrySet()) { - oprot.writeString(_iter4.getKey()); - oprot.writeI32(_iter4.getValue()); + oprot.writeString(_iter14.getKey()); + oprot.writeI32(_iter14.getValue()); } oprot.writeMapEnd(); } @@ -417,10 +417,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TStructTypeEntry st TTupleProtocol oprot = (TTupleProtocol) prot; { oprot.writeI32(struct.nameToTypePtr.size()); - for (Map.Entry _iter5 : struct.nameToTypePtr.entrySet()) + for (Map.Entry _iter15 : struct.nameToTypePtr.entrySet()) { - oprot.writeString(_iter5.getKey()); - oprot.writeI32(_iter5.getValue()); + oprot.writeString(_iter15.getKey()); + oprot.writeI32(_iter15.getValue()); } } } @@ -429,15 +429,15 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TStructTypeEntry st public void read(org.apache.thrift.protocol.TProtocol prot, TStructTypeEntry struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; { - org.apache.thrift.protocol.TMap _map6 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.I32, iprot.readI32()); - struct.nameToTypePtr = new HashMap(2*_map6.size); - for (int _i7 = 0; _i7 < _map6.size; ++_i7) + org.apache.thrift.protocol.TMap _map16 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.I32, iprot.readI32()); + struct.nameToTypePtr = new HashMap(2*_map16.size); + for (int _i17 = 0; _i17 < _map16.size; ++_i17) { - String _key8; // required - int _val9; // required - _key8 = iprot.readString(); - _val9 = iprot.readI32(); - struct.nameToTypePtr.put(_key8, _val9); + String _key18; // required + int _val19; // required + _key18 = iprot.readString(); + _val19 = iprot.readI32(); + struct.nameToTypePtr.put(_key18, _val19); } } struct.setNameToTypePtrIsSet(true); diff --git service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTableSchema.java service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTableSchema.java index 23238ad..ff5e54d 100644 --- service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTableSchema.java +++ service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTableSchema.java @@ -350,14 +350,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TTableSchema struct case 1: // COLUMNS if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { { - org.apache.thrift.protocol.TList _list28 = iprot.readListBegin(); - struct.columns = new ArrayList(_list28.size); - for (int _i29 = 0; _i29 < _list28.size; ++_i29) + org.apache.thrift.protocol.TList _list38 = iprot.readListBegin(); + struct.columns = new ArrayList(_list38.size); + for (int _i39 = 0; _i39 < _list38.size; ++_i39) { - TColumnDesc _elem30; // required - _elem30 = new TColumnDesc(); - _elem30.read(iprot); - struct.columns.add(_elem30); + TColumnDesc _elem40; // optional + _elem40 = new TColumnDesc(); + _elem40.read(iprot); + struct.columns.add(_elem40); } iprot.readListEnd(); } @@ -383,9 +383,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TTableSchema struc oprot.writeFieldBegin(COLUMNS_FIELD_DESC); { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.columns.size())); - for (TColumnDesc _iter31 : struct.columns) + for (TColumnDesc _iter41 : struct.columns) { - _iter31.write(oprot); + _iter41.write(oprot); } oprot.writeListEnd(); } @@ -410,9 +410,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TTableSchema struct TTupleProtocol oprot = (TTupleProtocol) prot; { oprot.writeI32(struct.columns.size()); - for (TColumnDesc _iter32 : struct.columns) + for (TColumnDesc _iter42 : struct.columns) { - _iter32.write(oprot); + _iter42.write(oprot); } } } @@ -421,14 +421,14 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TTableSchema struct public void read(org.apache.thrift.protocol.TProtocol prot, TTableSchema struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; { - org.apache.thrift.protocol.TList _list33 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); - struct.columns = new ArrayList(_list33.size); - for (int _i34 = 0; _i34 < _list33.size; ++_i34) + org.apache.thrift.protocol.TList _list43 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.columns = new ArrayList(_list43.size); + for (int _i44 = 0; _i44 < _list43.size; ++_i44) { - TColumnDesc _elem35; // required - _elem35 = new TColumnDesc(); - _elem35.read(iprot); - struct.columns.add(_elem35); + TColumnDesc _elem45; // optional + _elem45 = new TColumnDesc(); + _elem45.read(iprot); + struct.columns.add(_elem45); } } struct.setColumnsIsSet(true); diff --git service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeDesc.java service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeDesc.java index 973bddc..251f86a 100644 --- service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeDesc.java +++ service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeDesc.java @@ -350,14 +350,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TTypeDesc struct) t case 1: // TYPES if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { { - org.apache.thrift.protocol.TList _list20 = iprot.readListBegin(); - struct.types = new ArrayList(_list20.size); - for (int _i21 = 0; _i21 < _list20.size; ++_i21) + org.apache.thrift.protocol.TList _list30 = iprot.readListBegin(); + struct.types = new ArrayList(_list30.size); + for (int _i31 = 0; _i31 < _list30.size; ++_i31) { - TTypeEntry _elem22; // required - _elem22 = new TTypeEntry(); - _elem22.read(iprot); - struct.types.add(_elem22); + TTypeEntry _elem32; // optional + _elem32 = new TTypeEntry(); + _elem32.read(iprot); + struct.types.add(_elem32); } iprot.readListEnd(); } @@ -383,9 +383,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TTypeDesc struct) oprot.writeFieldBegin(TYPES_FIELD_DESC); { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.types.size())); - for (TTypeEntry _iter23 : struct.types) + for (TTypeEntry _iter33 : struct.types) { - _iter23.write(oprot); + _iter33.write(oprot); } oprot.writeListEnd(); } @@ -410,9 +410,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TTypeDesc struct) t TTupleProtocol oprot = (TTupleProtocol) prot; { oprot.writeI32(struct.types.size()); - for (TTypeEntry _iter24 : struct.types) + for (TTypeEntry _iter34 : struct.types) { - _iter24.write(oprot); + _iter34.write(oprot); } } } @@ -421,14 +421,14 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TTypeDesc struct) t public void read(org.apache.thrift.protocol.TProtocol prot, TTypeDesc struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; { - org.apache.thrift.protocol.TList _list25 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); - struct.types = new ArrayList(_list25.size); - for (int _i26 = 0; _i26 < _list25.size; ++_i26) + org.apache.thrift.protocol.TList _list35 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.types = new ArrayList(_list35.size); + for (int _i36 = 0; _i36 < _list35.size; ++_i36) { - TTypeEntry _elem27; // required - _elem27 = new TTypeEntry(); - _elem27.read(iprot); - struct.types.add(_elem27); + TTypeEntry _elem37; // optional + _elem37 = new TTypeEntry(); + _elem37.read(iprot); + struct.types.add(_elem37); } } struct.setTypesIsSet(true); diff --git service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeId.java service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeId.java index aecc59e..be70a3a 100644 --- service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeId.java +++ service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeId.java @@ -29,7 +29,8 @@ USER_DEFINED_TYPE(14), DECIMAL_TYPE(15), NULL_TYPE(16), - DATE_TYPE(17); + DATE_TYPE(17), + VARCHAR_TYPE(18); private final int value; @@ -86,6 +87,8 @@ public static TTypeId findByValue(int value) { return NULL_TYPE; case 17: return DATE_TYPE; + case 18: + return VARCHAR_TYPE; default: return null; } diff --git service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeQualifierValue.java service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeQualifierValue.java new file mode 100644 index 0000000..8c40687 --- /dev/null +++ service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeQualifierValue.java @@ -0,0 +1,361 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hive.service.cli.thrift; + +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TTypeQualifierValue extends org.apache.thrift.TUnion { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TTypeQualifierValue"); + private static final org.apache.thrift.protocol.TField I32_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("i32Value", org.apache.thrift.protocol.TType.I32, (short)1); + private static final org.apache.thrift.protocol.TField STRING_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("stringValue", org.apache.thrift.protocol.TType.STRING, (short)2); + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + I32_VALUE((short)1, "i32Value"), + STRING_VALUE((short)2, "stringValue"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // I32_VALUE + return I32_VALUE; + case 2: // STRING_VALUE + return STRING_VALUE; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.I32_VALUE, new org.apache.thrift.meta_data.FieldMetaData("i32Value", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + tmpMap.put(_Fields.STRING_VALUE, new org.apache.thrift.meta_data.FieldMetaData("stringValue", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TTypeQualifierValue.class, metaDataMap); + } + + public TTypeQualifierValue() { + super(); + } + + public TTypeQualifierValue(_Fields setField, Object value) { + super(setField, value); + } + + public TTypeQualifierValue(TTypeQualifierValue other) { + super(other); + } + public TTypeQualifierValue deepCopy() { + return new TTypeQualifierValue(this); + } + + public static TTypeQualifierValue i32Value(int value) { + TTypeQualifierValue x = new TTypeQualifierValue(); + x.setI32Value(value); + return x; + } + + public static TTypeQualifierValue stringValue(String value) { + TTypeQualifierValue x = new TTypeQualifierValue(); + x.setStringValue(value); + return x; + } + + + @Override + protected void checkType(_Fields setField, Object value) throws ClassCastException { + switch (setField) { + case I32_VALUE: + if (value instanceof Integer) { + break; + } + throw new ClassCastException("Was expecting value of type Integer for field 'i32Value', but got " + value.getClass().getSimpleName()); + case STRING_VALUE: + if (value instanceof String) { + break; + } + throw new ClassCastException("Was expecting value of type String for field 'stringValue', but got " + value.getClass().getSimpleName()); + default: + throw new IllegalArgumentException("Unknown field id " + setField); + } + } + + @Override + protected Object standardSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TField field) throws org.apache.thrift.TException { + _Fields setField = _Fields.findByThriftId(field.id); + if (setField != null) { + switch (setField) { + case I32_VALUE: + if (field.type == I32_VALUE_FIELD_DESC.type) { + Integer i32Value; + i32Value = iprot.readI32(); + return i32Value; + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + return null; + } + case STRING_VALUE: + if (field.type == STRING_VALUE_FIELD_DESC.type) { + String stringValue; + stringValue = iprot.readString(); + return stringValue; + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + return null; + } + default: + throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!"); + } + } else { + return null; + } + } + + @Override + protected void standardSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + switch (setField_) { + case I32_VALUE: + Integer i32Value = (Integer)value_; + oprot.writeI32(i32Value); + return; + case STRING_VALUE: + String stringValue = (String)value_; + oprot.writeString(stringValue); + return; + default: + throw new IllegalStateException("Cannot write union with unknown field " + setField_); + } + } + + @Override + protected Object tupleSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, short fieldID) throws org.apache.thrift.TException { + _Fields setField = _Fields.findByThriftId(fieldID); + if (setField != null) { + switch (setField) { + case I32_VALUE: + Integer i32Value; + i32Value = iprot.readI32(); + return i32Value; + case STRING_VALUE: + String stringValue; + stringValue = iprot.readString(); + return stringValue; + default: + throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!"); + } + } else { + throw new TProtocolException("Couldn't find a field with field id " + fieldID); + } + } + + @Override + protected void tupleSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + switch (setField_) { + case I32_VALUE: + Integer i32Value = (Integer)value_; + oprot.writeI32(i32Value); + return; + case STRING_VALUE: + String stringValue = (String)value_; + oprot.writeString(stringValue); + return; + default: + throw new IllegalStateException("Cannot write union with unknown field " + setField_); + } + } + + @Override + protected org.apache.thrift.protocol.TField getFieldDesc(_Fields setField) { + switch (setField) { + case I32_VALUE: + return I32_VALUE_FIELD_DESC; + case STRING_VALUE: + return STRING_VALUE_FIELD_DESC; + default: + throw new IllegalArgumentException("Unknown field id " + setField); + } + } + + @Override + protected org.apache.thrift.protocol.TStruct getStructDesc() { + return STRUCT_DESC; + } + + @Override + protected _Fields enumForId(short id) { + return _Fields.findByThriftIdOrThrow(id); + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + + public int getI32Value() { + if (getSetField() == _Fields.I32_VALUE) { + return (Integer)getFieldValue(); + } else { + throw new RuntimeException("Cannot get field 'i32Value' because union is currently set to " + getFieldDesc(getSetField()).name); + } + } + + public void setI32Value(int value) { + setField_ = _Fields.I32_VALUE; + value_ = value; + } + + public String getStringValue() { + if (getSetField() == _Fields.STRING_VALUE) { + return (String)getFieldValue(); + } else { + throw new RuntimeException("Cannot get field 'stringValue' because union is currently set to " + getFieldDesc(getSetField()).name); + } + } + + public void setStringValue(String value) { + if (value == null) throw new NullPointerException(); + setField_ = _Fields.STRING_VALUE; + value_ = value; + } + + public boolean isSetI32Value() { + return setField_ == _Fields.I32_VALUE; + } + + + public boolean isSetStringValue() { + return setField_ == _Fields.STRING_VALUE; + } + + + public boolean equals(Object other) { + if (other instanceof TTypeQualifierValue) { + return equals((TTypeQualifierValue)other); + } else { + return false; + } + } + + public boolean equals(TTypeQualifierValue other) { + return other != null && getSetField() == other.getSetField() && getFieldValue().equals(other.getFieldValue()); + } + + @Override + public int compareTo(TTypeQualifierValue other) { + int lastComparison = org.apache.thrift.TBaseHelper.compareTo(getSetField(), other.getSetField()); + if (lastComparison == 0) { + return org.apache.thrift.TBaseHelper.compareTo(getFieldValue(), other.getFieldValue()); + } + return lastComparison; + } + + + @Override + public int hashCode() { + HashCodeBuilder hcb = new HashCodeBuilder(); + hcb.append(this.getClass().getName()); + org.apache.thrift.TFieldIdEnum setField = getSetField(); + if (setField != null) { + hcb.append(setField.getThriftFieldId()); + Object value = getFieldValue(); + if (value instanceof org.apache.thrift.TEnum) { + hcb.append(((org.apache.thrift.TEnum)getFieldValue()).getValue()); + } else { + hcb.append(value); + } + } + return hcb.toHashCode(); + } + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + +} diff --git service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeQualifiers.java service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeQualifiers.java new file mode 100644 index 0000000..3935555 --- /dev/null +++ service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeQualifiers.java @@ -0,0 +1,450 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hive.service.cli.thrift; + +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TTypeQualifiers implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TTypeQualifiers"); + + private static final org.apache.thrift.protocol.TField QUALIFIERS_FIELD_DESC = new org.apache.thrift.protocol.TField("qualifiers", org.apache.thrift.protocol.TType.MAP, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new TTypeQualifiersStandardSchemeFactory()); + schemes.put(TupleScheme.class, new TTypeQualifiersTupleSchemeFactory()); + } + + private Map qualifiers; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + QUALIFIERS((short)1, "qualifiers"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // QUALIFIERS + return QUALIFIERS; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.QUALIFIERS, new org.apache.thrift.meta_data.FieldMetaData("qualifiers", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING), + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TTypeQualifierValue.class)))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TTypeQualifiers.class, metaDataMap); + } + + public TTypeQualifiers() { + } + + public TTypeQualifiers( + Map qualifiers) + { + this(); + this.qualifiers = qualifiers; + } + + /** + * Performs a deep copy on other. + */ + public TTypeQualifiers(TTypeQualifiers other) { + if (other.isSetQualifiers()) { + Map __this__qualifiers = new HashMap(); + for (Map.Entry other_element : other.qualifiers.entrySet()) { + + String other_element_key = other_element.getKey(); + TTypeQualifierValue other_element_value = other_element.getValue(); + + String __this__qualifiers_copy_key = other_element_key; + + TTypeQualifierValue __this__qualifiers_copy_value = new TTypeQualifierValue(other_element_value); + + __this__qualifiers.put(__this__qualifiers_copy_key, __this__qualifiers_copy_value); + } + this.qualifiers = __this__qualifiers; + } + } + + public TTypeQualifiers deepCopy() { + return new TTypeQualifiers(this); + } + + @Override + public void clear() { + this.qualifiers = null; + } + + public int getQualifiersSize() { + return (this.qualifiers == null) ? 0 : this.qualifiers.size(); + } + + public void putToQualifiers(String key, TTypeQualifierValue val) { + if (this.qualifiers == null) { + this.qualifiers = new HashMap(); + } + this.qualifiers.put(key, val); + } + + public Map getQualifiers() { + return this.qualifiers; + } + + public void setQualifiers(Map qualifiers) { + this.qualifiers = qualifiers; + } + + public void unsetQualifiers() { + this.qualifiers = null; + } + + /** Returns true if field qualifiers is set (has been assigned a value) and false otherwise */ + public boolean isSetQualifiers() { + return this.qualifiers != null; + } + + public void setQualifiersIsSet(boolean value) { + if (!value) { + this.qualifiers = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case QUALIFIERS: + if (value == null) { + unsetQualifiers(); + } else { + setQualifiers((Map)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case QUALIFIERS: + return getQualifiers(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case QUALIFIERS: + return isSetQualifiers(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof TTypeQualifiers) + return this.equals((TTypeQualifiers)that); + return false; + } + + public boolean equals(TTypeQualifiers that) { + if (that == null) + return false; + + boolean this_present_qualifiers = true && this.isSetQualifiers(); + boolean that_present_qualifiers = true && that.isSetQualifiers(); + if (this_present_qualifiers || that_present_qualifiers) { + if (!(this_present_qualifiers && that_present_qualifiers)) + return false; + if (!this.qualifiers.equals(that.qualifiers)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + HashCodeBuilder builder = new HashCodeBuilder(); + + boolean present_qualifiers = true && (isSetQualifiers()); + builder.append(present_qualifiers); + if (present_qualifiers) + builder.append(qualifiers); + + return builder.toHashCode(); + } + + public int compareTo(TTypeQualifiers other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + TTypeQualifiers typedOther = (TTypeQualifiers)other; + + lastComparison = Boolean.valueOf(isSetQualifiers()).compareTo(typedOther.isSetQualifiers()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetQualifiers()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.qualifiers, typedOther.qualifiers); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("TTypeQualifiers("); + boolean first = true; + + sb.append("qualifiers:"); + if (this.qualifiers == null) { + sb.append("null"); + } else { + sb.append(this.qualifiers); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + if (!isSetQualifiers()) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'qualifiers' is unset! Struct:" + toString()); + } + + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class TTypeQualifiersStandardSchemeFactory implements SchemeFactory { + public TTypeQualifiersStandardScheme getScheme() { + return new TTypeQualifiersStandardScheme(); + } + } + + private static class TTypeQualifiersStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, TTypeQualifiers struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // QUALIFIERS + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map0 = iprot.readMapBegin(); + struct.qualifiers = new HashMap(2*_map0.size); + for (int _i1 = 0; _i1 < _map0.size; ++_i1) + { + String _key2; // required + TTypeQualifierValue _val3; // required + _key2 = iprot.readString(); + _val3 = new TTypeQualifierValue(); + _val3.read(iprot); + struct.qualifiers.put(_key2, _val3); + } + iprot.readMapEnd(); + } + struct.setQualifiersIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, TTypeQualifiers struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.qualifiers != null) { + oprot.writeFieldBegin(QUALIFIERS_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, struct.qualifiers.size())); + for (Map.Entry _iter4 : struct.qualifiers.entrySet()) + { + oprot.writeString(_iter4.getKey()); + _iter4.getValue().write(oprot); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class TTypeQualifiersTupleSchemeFactory implements SchemeFactory { + public TTypeQualifiersTupleScheme getScheme() { + return new TTypeQualifiersTupleScheme(); + } + } + + private static class TTypeQualifiersTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, TTypeQualifiers struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + { + oprot.writeI32(struct.qualifiers.size()); + for (Map.Entry _iter5 : struct.qualifiers.entrySet()) + { + oprot.writeString(_iter5.getKey()); + _iter5.getValue().write(oprot); + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, TTypeQualifiers struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + { + org.apache.thrift.protocol.TMap _map6 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.qualifiers = new HashMap(2*_map6.size); + for (int _i7 = 0; _i7 < _map6.size; ++_i7) + { + String _key8; // required + TTypeQualifierValue _val9; // required + _key8 = iprot.readString(); + _val9 = new TTypeQualifierValue(); + _val9.read(iprot); + struct.qualifiers.put(_key8, _val9); + } + } + struct.setQualifiersIsSet(true); + } + } + +} + diff --git service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TUnionTypeEntry.java service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TUnionTypeEntry.java index 9254547..73dd45d 100644 --- service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TUnionTypeEntry.java +++ service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TUnionTypeEntry.java @@ -355,15 +355,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TUnionTypeEntry str case 1: // NAME_TO_TYPE_PTR if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { { - org.apache.thrift.protocol.TMap _map10 = iprot.readMapBegin(); - struct.nameToTypePtr = new HashMap(2*_map10.size); - for (int _i11 = 0; _i11 < _map10.size; ++_i11) + org.apache.thrift.protocol.TMap _map20 = iprot.readMapBegin(); + struct.nameToTypePtr = new HashMap(2*_map20.size); + for (int _i21 = 0; _i21 < _map20.size; ++_i21) { - String _key12; // required - int _val13; // required - _key12 = iprot.readString(); - _val13 = iprot.readI32(); - struct.nameToTypePtr.put(_key12, _val13); + String _key22; // required + int _val23; // required + _key22 = iprot.readString(); + _val23 = iprot.readI32(); + struct.nameToTypePtr.put(_key22, _val23); } iprot.readMapEnd(); } @@ -389,10 +389,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, TUnionTypeEntry st oprot.writeFieldBegin(NAME_TO_TYPE_PTR_FIELD_DESC); { oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.I32, struct.nameToTypePtr.size())); - for (Map.Entry _iter14 : struct.nameToTypePtr.entrySet()) + for (Map.Entry _iter24 : struct.nameToTypePtr.entrySet()) { - oprot.writeString(_iter14.getKey()); - oprot.writeI32(_iter14.getValue()); + oprot.writeString(_iter24.getKey()); + oprot.writeI32(_iter24.getValue()); } oprot.writeMapEnd(); } @@ -417,10 +417,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TUnionTypeEntry str TTupleProtocol oprot = (TTupleProtocol) prot; { oprot.writeI32(struct.nameToTypePtr.size()); - for (Map.Entry _iter15 : struct.nameToTypePtr.entrySet()) + for (Map.Entry _iter25 : struct.nameToTypePtr.entrySet()) { - oprot.writeString(_iter15.getKey()); - oprot.writeI32(_iter15.getValue()); + oprot.writeString(_iter25.getKey()); + oprot.writeI32(_iter25.getValue()); } } } @@ -429,15 +429,15 @@ public void write(org.apache.thrift.protocol.TProtocol prot, TUnionTypeEntry str public void read(org.apache.thrift.protocol.TProtocol prot, TUnionTypeEntry struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; { - org.apache.thrift.protocol.TMap _map16 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.I32, iprot.readI32()); - struct.nameToTypePtr = new HashMap(2*_map16.size); - for (int _i17 = 0; _i17 < _map16.size; ++_i17) + org.apache.thrift.protocol.TMap _map26 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.I32, iprot.readI32()); + struct.nameToTypePtr = new HashMap(2*_map26.size); + for (int _i27 = 0; _i27 < _map26.size; ++_i27) { - String _key18; // required - int _val19; // required - _key18 = iprot.readString(); - _val19 = iprot.readI32(); - struct.nameToTypePtr.put(_key18, _val19); + String _key28; // required + int _val29; // required + _key28 = iprot.readString(); + _val29 = iprot.readI32(); + struct.nameToTypePtr.put(_key28, _val29); } } struct.setNameToTypePtrIsSet(true); diff --git service/src/gen/thrift/gen-py/TCLIService/TCLIService-remote service/src/gen/thrift/gen-py/TCLIService/TCLIService-remote old mode 100644 new mode 100755 diff --git service/src/gen/thrift/gen-py/TCLIService/constants.py service/src/gen/thrift/gen-py/TCLIService/constants.py index a81196b..589ce88 100644 --- service/src/gen/thrift/gen-py/TCLIService/constants.py +++ service/src/gen/thrift/gen-py/TCLIService/constants.py @@ -23,6 +23,7 @@ 15, 16, 17, + 18, ]) COMPLEX_TYPES = set([ 10, @@ -53,4 +54,6 @@ 15 : "DECIMAL", 16 : "NULL", 17 : "DATE", + 18 : "VARCHAR", } +CHARACTER_MAXIMUM_LENGTH = "characterMaximumLength" diff --git service/src/gen/thrift/gen-py/TCLIService/ttypes.py service/src/gen/thrift/gen-py/TCLIService/ttypes.py index bf72ad3..cd40965 100644 --- service/src/gen/thrift/gen-py/TCLIService/ttypes.py +++ service/src/gen/thrift/gen-py/TCLIService/ttypes.py @@ -46,6 +46,7 @@ class TTypeId: DECIMAL_TYPE = 15 NULL_TYPE = 16 DATE_TYPE = 17 + VARCHAR_TYPE = 18 _VALUES_TO_NAMES = { 0: "BOOLEAN_TYPE", @@ -66,6 +67,7 @@ class TTypeId: 15: "DECIMAL_TYPE", 16: "NULL_TYPE", 17: "DATE_TYPE", + 18: "VARCHAR_TYPE", } _NAMES_TO_VALUES = { @@ -87,6 +89,7 @@ class TTypeId: "DECIMAL_TYPE": 15, "NULL_TYPE": 16, "DATE_TYPE": 17, + "VARCHAR_TYPE": 18, } class TStatusCode: @@ -352,19 +355,167 @@ class TFetchOrientation: } +class TTypeQualifierValue: + """ + Attributes: + - i32Value + - stringValue + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'i32Value', None, None, ), # 1 + (2, TType.STRING, 'stringValue', None, None, ), # 2 + ) + + def __init__(self, i32Value=None, stringValue=None,): + self.i32Value = i32Value + self.stringValue = stringValue + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.i32Value = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.stringValue = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('TTypeQualifierValue') + if self.i32Value is not None: + oprot.writeFieldBegin('i32Value', TType.I32, 1) + oprot.writeI32(self.i32Value) + oprot.writeFieldEnd() + if self.stringValue is not None: + oprot.writeFieldBegin('stringValue', TType.STRING, 2) + oprot.writeString(self.stringValue) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class TTypeQualifiers: + """ + Attributes: + - qualifiers + """ + + thrift_spec = ( + None, # 0 + (1, TType.MAP, 'qualifiers', (TType.STRING,None,TType.STRUCT,(TTypeQualifierValue, TTypeQualifierValue.thrift_spec)), None, ), # 1 + ) + + def __init__(self, qualifiers=None,): + self.qualifiers = qualifiers + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.MAP: + self.qualifiers = {} + (_ktype1, _vtype2, _size0 ) = iprot.readMapBegin() + for _i4 in xrange(_size0): + _key5 = iprot.readString(); + _val6 = TTypeQualifierValue() + _val6.read(iprot) + self.qualifiers[_key5] = _val6 + iprot.readMapEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('TTypeQualifiers') + if self.qualifiers is not None: + oprot.writeFieldBegin('qualifiers', TType.MAP, 1) + oprot.writeMapBegin(TType.STRING, TType.STRUCT, len(self.qualifiers)) + for kiter7,viter8 in self.qualifiers.items(): + oprot.writeString(kiter7) + viter8.write(oprot) + oprot.writeMapEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.qualifiers is None: + raise TProtocol.TProtocolException(message='Required field qualifiers is unset!') + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + class TPrimitiveTypeEntry: """ Attributes: - type + - typeQualifiers """ thrift_spec = ( None, # 0 (1, TType.I32, 'type', None, None, ), # 1 + (2, TType.STRUCT, 'typeQualifiers', (TTypeQualifiers, TTypeQualifiers.thrift_spec), None, ), # 2 ) - def __init__(self, type=None,): + def __init__(self, type=None, typeQualifiers=None,): self.type = type + self.typeQualifiers = typeQualifiers def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: @@ -380,6 +531,12 @@ def read(self, iprot): self.type = iprot.readI32(); else: iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.typeQualifiers = TTypeQualifiers() + self.typeQualifiers.read(iprot) + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -394,6 +551,10 @@ def write(self, oprot): oprot.writeFieldBegin('type', TType.I32, 1) oprot.writeI32(self.type) oprot.writeFieldEnd() + if self.typeQualifiers is not None: + oprot.writeFieldBegin('typeQualifiers', TType.STRUCT, 2) + self.typeQualifiers.write(oprot) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -578,11 +739,11 @@ def read(self, iprot): if fid == 1: if ftype == TType.MAP: self.nameToTypePtr = {} - (_ktype1, _vtype2, _size0 ) = iprot.readMapBegin() - for _i4 in xrange(_size0): - _key5 = iprot.readString(); - _val6 = iprot.readI32(); - self.nameToTypePtr[_key5] = _val6 + (_ktype10, _vtype11, _size9 ) = iprot.readMapBegin() + for _i13 in xrange(_size9): + _key14 = iprot.readString(); + _val15 = iprot.readI32(); + self.nameToTypePtr[_key14] = _val15 iprot.readMapEnd() else: iprot.skip(ftype) @@ -599,9 +760,9 @@ def write(self, oprot): if self.nameToTypePtr is not None: oprot.writeFieldBegin('nameToTypePtr', TType.MAP, 1) oprot.writeMapBegin(TType.STRING, TType.I32, len(self.nameToTypePtr)) - for kiter7,viter8 in self.nameToTypePtr.items(): - oprot.writeString(kiter7) - oprot.writeI32(viter8) + for kiter16,viter17 in self.nameToTypePtr.items(): + oprot.writeString(kiter16) + oprot.writeI32(viter17) oprot.writeMapEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -650,11 +811,11 @@ def read(self, iprot): if fid == 1: if ftype == TType.MAP: self.nameToTypePtr = {} - (_ktype10, _vtype11, _size9 ) = iprot.readMapBegin() - for _i13 in xrange(_size9): - _key14 = iprot.readString(); - _val15 = iprot.readI32(); - self.nameToTypePtr[_key14] = _val15 + (_ktype19, _vtype20, _size18 ) = iprot.readMapBegin() + for _i22 in xrange(_size18): + _key23 = iprot.readString(); + _val24 = iprot.readI32(); + self.nameToTypePtr[_key23] = _val24 iprot.readMapEnd() else: iprot.skip(ftype) @@ -671,9 +832,9 @@ def write(self, oprot): if self.nameToTypePtr is not None: oprot.writeFieldBegin('nameToTypePtr', TType.MAP, 1) oprot.writeMapBegin(TType.STRING, TType.I32, len(self.nameToTypePtr)) - for kiter16,viter17 in self.nameToTypePtr.items(): - oprot.writeString(kiter16) - oprot.writeI32(viter17) + for kiter25,viter26 in self.nameToTypePtr.items(): + oprot.writeString(kiter25) + oprot.writeI32(viter26) oprot.writeMapEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -910,11 +1071,11 @@ def read(self, iprot): if fid == 1: if ftype == TType.LIST: self.types = [] - (_etype21, _size18) = iprot.readListBegin() - for _i22 in xrange(_size18): - _elem23 = TTypeEntry() - _elem23.read(iprot) - self.types.append(_elem23) + (_etype30, _size27) = iprot.readListBegin() + for _i31 in xrange(_size27): + _elem32 = TTypeEntry() + _elem32.read(iprot) + self.types.append(_elem32) iprot.readListEnd() else: iprot.skip(ftype) @@ -931,8 +1092,8 @@ def write(self, oprot): if self.types is not None: oprot.writeFieldBegin('types', TType.LIST, 1) oprot.writeListBegin(TType.STRUCT, len(self.types)) - for iter24 in self.types: - iter24.write(oprot) + for iter33 in self.types: + iter33.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -1084,11 +1245,11 @@ def read(self, iprot): if fid == 1: if ftype == TType.LIST: self.columns = [] - (_etype28, _size25) = iprot.readListBegin() - for _i29 in xrange(_size25): - _elem30 = TColumnDesc() - _elem30.read(iprot) - self.columns.append(_elem30) + (_etype37, _size34) = iprot.readListBegin() + for _i38 in xrange(_size34): + _elem39 = TColumnDesc() + _elem39.read(iprot) + self.columns.append(_elem39) iprot.readListEnd() else: iprot.skip(ftype) @@ -1105,8 +1266,8 @@ def write(self, oprot): if self.columns is not None: oprot.writeFieldBegin('columns', TType.LIST, 1) oprot.writeListBegin(TType.STRUCT, len(self.columns)) - for iter31 in self.columns: - iter31.write(oprot) + for iter40 in self.columns: + iter40.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -1593,77 +1754,77 @@ def read(self, iprot): if fid == 1: if ftype == TType.LIST: self.boolColumn = [] - (_etype35, _size32) = iprot.readListBegin() - for _i36 in xrange(_size32): - _elem37 = TBoolValue() - _elem37.read(iprot) - self.boolColumn.append(_elem37) + (_etype44, _size41) = iprot.readListBegin() + for _i45 in xrange(_size41): + _elem46 = TBoolValue() + _elem46.read(iprot) + self.boolColumn.append(_elem46) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 2: if ftype == TType.LIST: self.byteColumn = [] - (_etype41, _size38) = iprot.readListBegin() - for _i42 in xrange(_size38): - _elem43 = TByteValue() - _elem43.read(iprot) - self.byteColumn.append(_elem43) + (_etype50, _size47) = iprot.readListBegin() + for _i51 in xrange(_size47): + _elem52 = TByteValue() + _elem52.read(iprot) + self.byteColumn.append(_elem52) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 3: if ftype == TType.LIST: self.i16Column = [] - (_etype47, _size44) = iprot.readListBegin() - for _i48 in xrange(_size44): - _elem49 = TI16Value() - _elem49.read(iprot) - self.i16Column.append(_elem49) + (_etype56, _size53) = iprot.readListBegin() + for _i57 in xrange(_size53): + _elem58 = TI16Value() + _elem58.read(iprot) + self.i16Column.append(_elem58) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 4: if ftype == TType.LIST: self.i32Column = [] - (_etype53, _size50) = iprot.readListBegin() - for _i54 in xrange(_size50): - _elem55 = TI32Value() - _elem55.read(iprot) - self.i32Column.append(_elem55) + (_etype62, _size59) = iprot.readListBegin() + for _i63 in xrange(_size59): + _elem64 = TI32Value() + _elem64.read(iprot) + self.i32Column.append(_elem64) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 5: if ftype == TType.LIST: self.i64Column = [] - (_etype59, _size56) = iprot.readListBegin() - for _i60 in xrange(_size56): - _elem61 = TI64Value() - _elem61.read(iprot) - self.i64Column.append(_elem61) + (_etype68, _size65) = iprot.readListBegin() + for _i69 in xrange(_size65): + _elem70 = TI64Value() + _elem70.read(iprot) + self.i64Column.append(_elem70) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 6: if ftype == TType.LIST: self.doubleColumn = [] - (_etype65, _size62) = iprot.readListBegin() - for _i66 in xrange(_size62): - _elem67 = TDoubleValue() - _elem67.read(iprot) - self.doubleColumn.append(_elem67) + (_etype74, _size71) = iprot.readListBegin() + for _i75 in xrange(_size71): + _elem76 = TDoubleValue() + _elem76.read(iprot) + self.doubleColumn.append(_elem76) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 7: if ftype == TType.LIST: self.stringColumn = [] - (_etype71, _size68) = iprot.readListBegin() - for _i72 in xrange(_size68): - _elem73 = TStringValue() - _elem73.read(iprot) - self.stringColumn.append(_elem73) + (_etype80, _size77) = iprot.readListBegin() + for _i81 in xrange(_size77): + _elem82 = TStringValue() + _elem82.read(iprot) + self.stringColumn.append(_elem82) iprot.readListEnd() else: iprot.skip(ftype) @@ -1680,50 +1841,50 @@ def write(self, oprot): if self.boolColumn is not None: oprot.writeFieldBegin('boolColumn', TType.LIST, 1) oprot.writeListBegin(TType.STRUCT, len(self.boolColumn)) - for iter74 in self.boolColumn: - iter74.write(oprot) + for iter83 in self.boolColumn: + iter83.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.byteColumn is not None: oprot.writeFieldBegin('byteColumn', TType.LIST, 2) oprot.writeListBegin(TType.STRUCT, len(self.byteColumn)) - for iter75 in self.byteColumn: - iter75.write(oprot) + for iter84 in self.byteColumn: + iter84.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.i16Column is not None: oprot.writeFieldBegin('i16Column', TType.LIST, 3) oprot.writeListBegin(TType.STRUCT, len(self.i16Column)) - for iter76 in self.i16Column: - iter76.write(oprot) + for iter85 in self.i16Column: + iter85.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.i32Column is not None: oprot.writeFieldBegin('i32Column', TType.LIST, 4) oprot.writeListBegin(TType.STRUCT, len(self.i32Column)) - for iter77 in self.i32Column: - iter77.write(oprot) + for iter86 in self.i32Column: + iter86.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.i64Column is not None: oprot.writeFieldBegin('i64Column', TType.LIST, 5) oprot.writeListBegin(TType.STRUCT, len(self.i64Column)) - for iter78 in self.i64Column: - iter78.write(oprot) + for iter87 in self.i64Column: + iter87.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.doubleColumn is not None: oprot.writeFieldBegin('doubleColumn', TType.LIST, 6) oprot.writeListBegin(TType.STRUCT, len(self.doubleColumn)) - for iter79 in self.doubleColumn: - iter79.write(oprot) + for iter88 in self.doubleColumn: + iter88.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.stringColumn is not None: oprot.writeFieldBegin('stringColumn', TType.LIST, 7) oprot.writeListBegin(TType.STRUCT, len(self.stringColumn)) - for iter80 in self.stringColumn: - iter80.write(oprot) + for iter89 in self.stringColumn: + iter89.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -1909,11 +2070,11 @@ def read(self, iprot): if fid == 1: if ftype == TType.LIST: self.colVals = [] - (_etype84, _size81) = iprot.readListBegin() - for _i85 in xrange(_size81): - _elem86 = TColumnValue() - _elem86.read(iprot) - self.colVals.append(_elem86) + (_etype93, _size90) = iprot.readListBegin() + for _i94 in xrange(_size90): + _elem95 = TColumnValue() + _elem95.read(iprot) + self.colVals.append(_elem95) iprot.readListEnd() else: iprot.skip(ftype) @@ -1930,8 +2091,8 @@ def write(self, oprot): if self.colVals is not None: oprot.writeFieldBegin('colVals', TType.LIST, 1) oprot.writeListBegin(TType.STRUCT, len(self.colVals)) - for iter87 in self.colVals: - iter87.write(oprot) + for iter96 in self.colVals: + iter96.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -1991,22 +2152,22 @@ def read(self, iprot): elif fid == 2: if ftype == TType.LIST: self.rows = [] - (_etype91, _size88) = iprot.readListBegin() - for _i92 in xrange(_size88): - _elem93 = TRow() - _elem93.read(iprot) - self.rows.append(_elem93) + (_etype100, _size97) = iprot.readListBegin() + for _i101 in xrange(_size97): + _elem102 = TRow() + _elem102.read(iprot) + self.rows.append(_elem102) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 3: if ftype == TType.LIST: self.columns = [] - (_etype97, _size94) = iprot.readListBegin() - for _i98 in xrange(_size94): - _elem99 = TColumn() - _elem99.read(iprot) - self.columns.append(_elem99) + (_etype106, _size103) = iprot.readListBegin() + for _i107 in xrange(_size103): + _elem108 = TColumn() + _elem108.read(iprot) + self.columns.append(_elem108) iprot.readListEnd() else: iprot.skip(ftype) @@ -2027,15 +2188,15 @@ def write(self, oprot): if self.rows is not None: oprot.writeFieldBegin('rows', TType.LIST, 2) oprot.writeListBegin(TType.STRUCT, len(self.rows)) - for iter100 in self.rows: - iter100.write(oprot) + for iter109 in self.rows: + iter109.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.columns is not None: oprot.writeFieldBegin('columns', TType.LIST, 3) oprot.writeListBegin(TType.STRUCT, len(self.columns)) - for iter101 in self.columns: - iter101.write(oprot) + for iter110 in self.columns: + iter110.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -2103,10 +2264,10 @@ def read(self, iprot): elif fid == 2: if ftype == TType.LIST: self.infoMessages = [] - (_etype105, _size102) = iprot.readListBegin() - for _i106 in xrange(_size102): - _elem107 = iprot.readString(); - self.infoMessages.append(_elem107) + (_etype114, _size111) = iprot.readListBegin() + for _i115 in xrange(_size111): + _elem116 = iprot.readString(); + self.infoMessages.append(_elem116) iprot.readListEnd() else: iprot.skip(ftype) @@ -2142,8 +2303,8 @@ def write(self, oprot): if self.infoMessages is not None: oprot.writeFieldBegin('infoMessages', TType.LIST, 2) oprot.writeListBegin(TType.STRING, len(self.infoMessages)) - for iter108 in self.infoMessages: - oprot.writeString(iter108) + for iter117 in self.infoMessages: + oprot.writeString(iter117) oprot.writeListEnd() oprot.writeFieldEnd() if self.sqlState is not None: @@ -2470,11 +2631,11 @@ def read(self, iprot): elif fid == 4: if ftype == TType.MAP: self.configuration = {} - (_ktype110, _vtype111, _size109 ) = iprot.readMapBegin() - for _i113 in xrange(_size109): - _key114 = iprot.readString(); - _val115 = iprot.readString(); - self.configuration[_key114] = _val115 + (_ktype119, _vtype120, _size118 ) = iprot.readMapBegin() + for _i122 in xrange(_size118): + _key123 = iprot.readString(); + _val124 = iprot.readString(); + self.configuration[_key123] = _val124 iprot.readMapEnd() else: iprot.skip(ftype) @@ -2503,9 +2664,9 @@ def write(self, oprot): if self.configuration is not None: oprot.writeFieldBegin('configuration', TType.MAP, 4) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.configuration)) - for kiter116,viter117 in self.configuration.items(): - oprot.writeString(kiter116) - oprot.writeString(viter117) + for kiter125,viter126 in self.configuration.items(): + oprot.writeString(kiter125) + oprot.writeString(viter126) oprot.writeMapEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -2580,11 +2741,11 @@ def read(self, iprot): elif fid == 4: if ftype == TType.MAP: self.configuration = {} - (_ktype119, _vtype120, _size118 ) = iprot.readMapBegin() - for _i122 in xrange(_size118): - _key123 = iprot.readString(); - _val124 = iprot.readString(); - self.configuration[_key123] = _val124 + (_ktype128, _vtype129, _size127 ) = iprot.readMapBegin() + for _i131 in xrange(_size127): + _key132 = iprot.readString(); + _val133 = iprot.readString(); + self.configuration[_key132] = _val133 iprot.readMapEnd() else: iprot.skip(ftype) @@ -2613,9 +2774,9 @@ def write(self, oprot): if self.configuration is not None: oprot.writeFieldBegin('configuration', TType.MAP, 4) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.configuration)) - for kiter125,viter126 in self.configuration.items(): - oprot.writeString(kiter125) - oprot.writeString(viter126) + for kiter134,viter135 in self.configuration.items(): + oprot.writeString(kiter134) + oprot.writeString(viter135) oprot.writeMapEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -3084,11 +3245,11 @@ def read(self, iprot): elif fid == 3: if ftype == TType.MAP: self.confOverlay = {} - (_ktype128, _vtype129, _size127 ) = iprot.readMapBegin() - for _i131 in xrange(_size127): - _key132 = iprot.readString(); - _val133 = iprot.readString(); - self.confOverlay[_key132] = _val133 + (_ktype137, _vtype138, _size136 ) = iprot.readMapBegin() + for _i140 in xrange(_size136): + _key141 = iprot.readString(); + _val142 = iprot.readString(); + self.confOverlay[_key141] = _val142 iprot.readMapEnd() else: iprot.skip(ftype) @@ -3113,9 +3274,9 @@ def write(self, oprot): if self.confOverlay is not None: oprot.writeFieldBegin('confOverlay', TType.MAP, 3) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.confOverlay)) - for kiter134,viter135 in self.confOverlay.items(): - oprot.writeString(kiter134) - oprot.writeString(viter135) + for kiter143,viter144 in self.confOverlay.items(): + oprot.writeString(kiter143) + oprot.writeString(viter144) oprot.writeMapEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -3716,10 +3877,10 @@ def read(self, iprot): elif fid == 5: if ftype == TType.LIST: self.tableTypes = [] - (_etype139, _size136) = iprot.readListBegin() - for _i140 in xrange(_size136): - _elem141 = iprot.readString(); - self.tableTypes.append(_elem141) + (_etype148, _size145) = iprot.readListBegin() + for _i149 in xrange(_size145): + _elem150 = iprot.readString(); + self.tableTypes.append(_elem150) iprot.readListEnd() else: iprot.skip(ftype) @@ -3752,8 +3913,8 @@ def write(self, oprot): if self.tableTypes is not None: oprot.writeFieldBegin('tableTypes', TType.LIST, 5) oprot.writeListBegin(TType.STRING, len(self.tableTypes)) - for iter142 in self.tableTypes: - oprot.writeString(iter142) + for iter151 in self.tableTypes: + oprot.writeString(iter151) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() diff --git service/src/gen/thrift/gen-py/hive_service/ThriftHive-remote service/src/gen/thrift/gen-py/hive_service/ThriftHive-remote old mode 100644 new mode 100755 diff --git service/src/gen/thrift/gen-rb/t_c_l_i_service_constants.rb service/src/gen/thrift/gen-rb/t_c_l_i_service_constants.rb index f576689..8c341c8 100644 --- service/src/gen/thrift/gen-rb/t_c_l_i_service_constants.rb +++ service/src/gen/thrift/gen-rb/t_c_l_i_service_constants.rb @@ -21,6 +21,7 @@ PRIMITIVE_TYPES = Set.new([ 15, 16, 17, + 18, ]) COMPLEX_TYPES = Set.new([ @@ -54,5 +55,8 @@ TYPE_NAMES = { 15 => %q"DECIMAL", 16 => %q"NULL", 17 => %q"DATE", + 18 => %q"VARCHAR", } +CHARACTER_MAXIMUM_LENGTH = %q"characterMaximumLength" + diff --git service/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb service/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb index 1da48ca..6815412 100644 --- service/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb +++ service/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb @@ -31,8 +31,9 @@ module TTypeId DECIMAL_TYPE = 15 NULL_TYPE = 16 DATE_TYPE = 17 - VALUE_MAP = {0 => "BOOLEAN_TYPE", 1 => "TINYINT_TYPE", 2 => "SMALLINT_TYPE", 3 => "INT_TYPE", 4 => "BIGINT_TYPE", 5 => "FLOAT_TYPE", 6 => "DOUBLE_TYPE", 7 => "STRING_TYPE", 8 => "TIMESTAMP_TYPE", 9 => "BINARY_TYPE", 10 => "ARRAY_TYPE", 11 => "MAP_TYPE", 12 => "STRUCT_TYPE", 13 => "UNION_TYPE", 14 => "USER_DEFINED_TYPE", 15 => "DECIMAL_TYPE", 16 => "NULL_TYPE", 17 => "DATE_TYPE"} - VALID_VALUES = Set.new([BOOLEAN_TYPE, TINYINT_TYPE, SMALLINT_TYPE, INT_TYPE, BIGINT_TYPE, FLOAT_TYPE, DOUBLE_TYPE, STRING_TYPE, TIMESTAMP_TYPE, BINARY_TYPE, ARRAY_TYPE, MAP_TYPE, STRUCT_TYPE, UNION_TYPE, USER_DEFINED_TYPE, DECIMAL_TYPE, NULL_TYPE, DATE_TYPE]).freeze + VARCHAR_TYPE = 18 + VALUE_MAP = {0 => "BOOLEAN_TYPE", 1 => "TINYINT_TYPE", 2 => "SMALLINT_TYPE", 3 => "INT_TYPE", 4 => "BIGINT_TYPE", 5 => "FLOAT_TYPE", 6 => "DOUBLE_TYPE", 7 => "STRING_TYPE", 8 => "TIMESTAMP_TYPE", 9 => "BINARY_TYPE", 10 => "ARRAY_TYPE", 11 => "MAP_TYPE", 12 => "STRUCT_TYPE", 13 => "UNION_TYPE", 14 => "USER_DEFINED_TYPE", 15 => "DECIMAL_TYPE", 16 => "NULL_TYPE", 17 => "DATE_TYPE", 18 => "VARCHAR_TYPE"} + VALID_VALUES = Set.new([BOOLEAN_TYPE, TINYINT_TYPE, SMALLINT_TYPE, INT_TYPE, BIGINT_TYPE, FLOAT_TYPE, DOUBLE_TYPE, STRING_TYPE, TIMESTAMP_TYPE, BINARY_TYPE, ARRAY_TYPE, MAP_TYPE, STRUCT_TYPE, UNION_TYPE, USER_DEFINED_TYPE, DECIMAL_TYPE, NULL_TYPE, DATE_TYPE, VARCHAR_TYPE]).freeze end module TStatusCode @@ -134,12 +135,60 @@ module TFetchOrientation VALID_VALUES = Set.new([FETCH_NEXT, FETCH_PRIOR, FETCH_RELATIVE, FETCH_ABSOLUTE, FETCH_FIRST, FETCH_LAST]).freeze end +class TTypeQualifierValue < ::Thrift::Union + include ::Thrift::Struct_Union + class << self + def i32Value(val) + TTypeQualifierValue.new(:i32Value, val) + end + + def stringValue(val) + TTypeQualifierValue.new(:stringValue, val) + end + end + + I32VALUE = 1 + STRINGVALUE = 2 + + FIELDS = { + I32VALUE => {:type => ::Thrift::Types::I32, :name => 'i32Value', :optional => true}, + STRINGVALUE => {:type => ::Thrift::Types::STRING, :name => 'stringValue', :optional => true} + } + + def struct_fields; FIELDS; end + + def validate + raise(StandardError, 'Union fields are not set.') if get_set_field.nil? || get_value.nil? + end + + ::Thrift::Union.generate_accessors self +end + +class TTypeQualifiers + include ::Thrift::Struct, ::Thrift::Struct_Union + QUALIFIERS = 1 + + FIELDS = { + QUALIFIERS => {:type => ::Thrift::Types::MAP, :name => 'qualifiers', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRUCT, :class => ::TTypeQualifierValue}} + } + + def struct_fields; FIELDS; end + + def validate + raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field qualifiers is unset!') unless @qualifiers + end + + ::Thrift::Struct.generate_accessors self +end + class TPrimitiveTypeEntry include ::Thrift::Struct, ::Thrift::Struct_Union TYPE = 1 + TYPEQUALIFIERS = 2 FIELDS = { - TYPE => {:type => ::Thrift::Types::I32, :name => 'type', :enum_class => ::TTypeId} + TYPE => {:type => ::Thrift::Types::I32, :name => 'type', :enum_class => ::TTypeId}, + TYPEQUALIFIERS => {:type => ::Thrift::Types::STRUCT, :name => 'typeQualifiers', :class => ::TTypeQualifiers, :optional => true} } def struct_fields; FIELDS; end diff --git service/src/java/org/apache/hive/service/cli/ColumnDescriptor.java service/src/java/org/apache/hive/service/cli/ColumnDescriptor.java index d702723..8324f95 100644 --- service/src/java/org/apache/hive/service/cli/ColumnDescriptor.java +++ service/src/java/org/apache/hive/service/cli/ColumnDescriptor.java @@ -50,11 +50,13 @@ public ColumnDescriptor(TColumnDesc tColumnDesc) { public ColumnDescriptor(FieldSchema column, int position) { name = column.getName(); comment = column.getComment(); - type = new TypeDescriptor(column.getType()); + type = new TypeDescriptor(column); this.position = position; } public static ColumnDescriptor newPrimitiveColumnDescriptor(String name, String comment, Type type, int position) { + // Current usage looks like it's only for metadata columns, but if that changes then + // this method may need to require a type qualifiers aruments. return new ColumnDescriptor(name, comment, new TypeDescriptor(type), position); } diff --git service/src/java/org/apache/hive/service/cli/ColumnValue.java service/src/java/org/apache/hive/service/cli/ColumnValue.java index 75b5407..905ba01 100644 --- service/src/java/org/apache/hive/service/cli/ColumnValue.java +++ service/src/java/org/apache/hive/service/cli/ColumnValue.java @@ -22,6 +22,7 @@ import java.sql.Timestamp; import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hive.service.cli.thrift.TBoolValue; import org.apache.hive.service.cli.thrift.TByteValue; import org.apache.hive.service.cli.thrift.TColumnValue; @@ -118,6 +119,14 @@ public static ColumnValue stringValue(String value) { return new ColumnValue(TColumnValue.stringVal(tStringValue)); } + public static ColumnValue stringValue(HiveVarchar value) { + TStringValue tStringValue = new TStringValue(); + if (value != null) { + tStringValue.setValue(value.toString()); + } + return new ColumnValue(TColumnValue.stringVal(tStringValue)); + } + public static ColumnValue dateValue(Date value) { TStringValue tStringValue = new TStringValue(); if (value != null) { @@ -160,6 +169,8 @@ public static ColumnValue newColumnValue(Type type, Object value) { return doubleValue((Double)value); case STRING_TYPE: return stringValue((String)value); + case VARCHAR_TYPE: + return stringValue((HiveVarchar)value); case DATE_TYPE: return dateValue((Date)value); case TIMESTAMP_TYPE: diff --git service/src/java/org/apache/hive/service/cli/Type.java service/src/java/org/apache/hive/service/cli/Type.java index 3423262..f414fca 100644 --- service/src/java/org/apache/hive/service/cli/Type.java +++ service/src/java/org/apache/hive/service/cli/Type.java @@ -54,6 +54,10 @@ STRING_TYPE("STRING", java.sql.Types.VARCHAR, TTypeId.STRING_TYPE), + VARCHAR_TYPE("VARCHAR", + java.sql.Types.VARCHAR, + TTypeId.VARCHAR_TYPE, + true, false, false), DATE_TYPE("DATE", java.sql.Types.DATE, TTypeId.DATE_TYPE), @@ -66,7 +70,7 @@ DECIMAL_TYPE("DECIMAL", java.sql.Types.DECIMAL, TTypeId.DECIMAL_TYPE, - false, false), + true, false, false), ARRAY_TYPE("ARRAY", java.sql.Types.VARCHAR, TTypeId.STRING_TYPE, @@ -91,26 +95,35 @@ private final String name; private final TTypeId tType; private final int javaSQLType; + private final boolean isQualified; private final boolean isComplex; private final boolean isCollection; - - Type(String name, int javaSQLType, TTypeId tType, boolean isComplex, boolean isCollection) { + Type(String name, int javaSQLType, TTypeId tType, boolean isQualified, boolean isComplex, boolean isCollection) { this.name = name; this.javaSQLType = javaSQLType; this.tType = tType; + this.isQualified = isQualified; this.isComplex = isComplex; this.isCollection = isCollection; } + Type(String name, int javaSQLType, TTypeId tType, boolean isComplex, boolean isCollection) { + this(name, javaSQLType, tType, false, isComplex, isCollection); + } + Type(String name, int javaSqlType, TTypeId tType) { - this(name, javaSqlType, tType, false, false); + this(name, javaSqlType, tType, false, false, false); } public boolean isPrimitiveType() { return !isComplex; } + public boolean isQualifiedType() { + return isQualified; + } + public boolean isComplexType() { return isComplex; } @@ -135,7 +148,7 @@ public static Type getType(String name) { for (Type type : values()) { if (name.equalsIgnoreCase(type.name)) { return type; - } else if (type.isComplexType()) { + } else if (type.isQualifiedType() || type.isComplexType()) { if (name.toUpperCase().startsWith(type.name)) { return type; } diff --git service/src/java/org/apache/hive/service/cli/TypeDescriptor.java service/src/java/org/apache/hive/service/cli/TypeDescriptor.java index b4817ad..8a838b1 100644 --- service/src/java/org/apache/hive/service/cli/TypeDescriptor.java +++ service/src/java/org/apache/hive/service/cli/TypeDescriptor.java @@ -20,6 +20,11 @@ import java.util.List; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.serde2.typeinfo.BaseTypeParams; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hive.service.cli.thrift.TPrimitiveTypeEntry; import org.apache.hive.service.cli.thrift.TTypeDesc; import org.apache.hive.service.cli.thrift.TTypeEntry; @@ -32,6 +37,7 @@ private final Type type; private String typeName = null; + private TypeQualifiers typeQualifiers = null; public TypeDescriptor(Type type) { this.type = type; @@ -41,21 +47,39 @@ public TypeDescriptor(TTypeDesc tTypeDesc) { List tTypeEntries = tTypeDesc.getTypes(); TPrimitiveTypeEntry top = tTypeEntries.get(0).getPrimitiveEntry(); this.type = Type.getType(top.getType()); + if (top.isSetTypeQualifiers()) { + setTypeQualifiers(TypeQualifiers.fromTTypeQualifiers(top.getTypeQualifiers())); + } } public TypeDescriptor(String typeName) { this.type = Type.getType(typeName); if (this.type.isComplexType()) { this.typeName = typeName; + } else if (this.type.isQualifiedType()) { + TypeInfo pti = TypeInfoFactory.getPrimitiveTypeInfo(typeName); + BaseTypeParams typeParams = ((PrimitiveTypeInfo) pti).getParameters(); + if (typeParams != null) { + setTypeQualifiers(TypeQualifiers.fromBaseTypeParams(typeParams)); + } } } + public TypeDescriptor(FieldSchema fieldSchema) { + this(fieldSchema.getType()); + } + public Type getType() { return type; } public TTypeDesc toTTypeDesc() { - TTypeEntry entry = TTypeEntry.primitiveEntry(new TPrimitiveTypeEntry(type.toTType())); + TPrimitiveTypeEntry primitiveEntry = new TPrimitiveTypeEntry(type.toTType()); + if (getTypeQualifiers() != null) { + primitiveEntry.setTypeQualifiers(getTypeQualifiers().toTTypeQualifiers()); + } + TTypeEntry entry = TTypeEntry.primitiveEntry(primitiveEntry); + TTypeDesc desc = new TTypeDesc(); desc.addToTypes(entry); return desc; @@ -68,4 +92,12 @@ public String getTypeName() { return type.getName(); } } + + public TypeQualifiers getTypeQualifiers() { + return typeQualifiers; + } + + public void setTypeQualifiers(TypeQualifiers typeQualifiers) { + this.typeQualifiers = typeQualifiers; + } } diff --git service/src/java/org/apache/hive/service/cli/TypeQualifiers.java service/src/java/org/apache/hive/service/cli/TypeQualifiers.java new file mode 100644 index 0000000..fd2f7d1 --- /dev/null +++ service/src/java/org/apache/hive/service/cli/TypeQualifiers.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.service.cli; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.hive.serde2.typeinfo.BaseTypeParams; +import org.apache.hive.service.cli.thrift.TCLIServiceConstants; +import org.apache.hive.service.cli.thrift.TTypeQualifierValue; +import org.apache.hive.service.cli.thrift.TTypeQualifiers; + +public class TypeQualifiers { + private Integer characterMaximumLength; + + public TypeQualifiers() {} + + public Integer getCharacterMaximumLength() { + return characterMaximumLength; + } + public void setCharacterMaximumLength(int characterMaximumLength) { + this.characterMaximumLength = characterMaximumLength; + } + + public TTypeQualifiers toTTypeQualifiers() { + TTypeQualifiers ret = null; + + Map qMap = new HashMap(); + if (getCharacterMaximumLength() != null) { + TTypeQualifierValue val = new TTypeQualifierValue(); + val.setI32Value(getCharacterMaximumLength().intValue()); + qMap.put(TCLIServiceConstants.CHARACTER_MAXIMUM_LENGTH, val); + } + + if (qMap.size() > 0) { + ret = new TTypeQualifiers(qMap); + } + + return ret; + } + + public static TypeQualifiers fromTTypeQualifiers(TTypeQualifiers ttq) { + TypeQualifiers ret = null; + if (ttq != null) { + ret = new TypeQualifiers(); + Map tqMap = ttq.getQualifiers(); + + if (tqMap.containsKey(TCLIServiceConstants.CHARACTER_MAXIMUM_LENGTH)) { + ret.setCharacterMaximumLength( + tqMap.get(TCLIServiceConstants.CHARACTER_MAXIMUM_LENGTH).getI32Value()); + } + } + return ret; + } + + public static TypeQualifiers fromBaseTypeParams(BaseTypeParams typeParams) { + TypeQualifiers ret = null; + if (typeParams != null) { + ret = new TypeQualifiers(); + if (typeParams.hasCharacterMaximumLength()) { + ret.setCharacterMaximumLength(typeParams.getCharacterMaximumLength()); + } + } + return ret; + } +}