diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index a5c7daf..a23d122 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -309,7 +309,7 @@ private boolean isViewTable(String dbName, String tblName) throws MetaException StringBuilder partSb = new StringBuilder(sbCapacity); // Assume db and table names are the same for all partition, that's what we're selecting for. for (Object partitionId : sqlResult) { - partSb.append((Long)partitionId).append(","); + partSb.append(extractSqlLong(partitionId)).append(","); } String partIds = trimCommaList(partSb); timingTrace(doTrace, queryText, start, queryTime); @@ -346,10 +346,10 @@ private boolean isViewTable(String dbName, String tblName) throws MetaException dbName = dbName.toLowerCase(); for (Object[] fields : sqlResult2) { // Here comes the ugly part... - long partitionId = (Long)fields[0]; - Long sdId = (Long)fields[1]; - Long colId = (Long)fields[2]; - Long serdeId = (Long)fields[3]; + long partitionId = extractSqlLong(fields[0]); + Long sdId = extractSqlLong(fields[1]); + Long colId = extractSqlLong(fields[2]); + Long serdeId = extractSqlLong(fields[3]); // A partition must have either everything set, or nothing set if it's a view. if (sdId == null || colId == null || serdeId == null) { if (isView == null) { @@ -502,7 +502,7 @@ public void apply(StorageDescriptor t, Object[] fields) { loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc() { private Long currentListId; private List currentList; - public void apply(StorageDescriptor t, Object[] fields) { + public void apply(StorageDescriptor t, Object[] fields) throws MetaException { if (!t.isSetSkewedInfo()) t.setSkewedInfo(new SkewedInfo()); // Note that this is not a typical list accumulator - there's no call to finalize // the last list. Instead we add list to SD first, as well as locally to add elements. @@ -511,7 +511,7 @@ public void apply(StorageDescriptor t, Object[] fields) { currentListId = null; t.getSkewedInfo().addToSkewedColValues(new ArrayList()); } else { - long fieldsListId = (Long)fields[1]; + long fieldsListId = extractSqlLong(fields[1]); if (currentListId == null || fieldsListId != currentListId) { currentList = new ArrayList(); currentListId = fieldsListId; @@ -539,7 +539,7 @@ public void apply(StorageDescriptor t, Object[] fields) { loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc() { private Long currentListId; private List currentList; - public void apply(StorageDescriptor t, Object[] fields) { + public void apply(StorageDescriptor t, Object[] fields) throws MetaException { if (!t.isSetSkewedInfo()) { SkewedInfo skewedInfo = new SkewedInfo(); skewedInfo.setSkewedColValueLocationMaps(new HashMap, String>()); @@ -552,7 +552,7 @@ public void apply(StorageDescriptor t, Object[] fields) { currentList = new ArrayList(); // left outer join produced a list with no values currentListId = null; } else { - long fieldsListId = (Long)fields[1]; + long fieldsListId = extractSqlLong(fields[1]); if (currentListId == null || fieldsListId != currentListId) { currentList = new ArrayList(); currentListId = fieldsListId; @@ -589,6 +589,14 @@ public void apply(SerDeInfo t, Object[] fields) { return orderedResult; } + private Long extractSqlLong(Object obj) throws MetaException { + if (obj == null) return null; + if (!(obj instanceof Number)) { + throw new MetaException("Expected numeric type but got " + obj.getClass().getName()); + } + return ((Number)obj).longValue(); + } + private void timingTrace(boolean doTrace, String queryText, long start, long queryTime) { if (!doTrace) return; LOG.debug("Direct SQL query in " + (queryTime - start) / 1000000.0 + "ms + " + @@ -604,9 +612,10 @@ private static Boolean extractSqlBoolean(Object value) throws MetaException { if (value instanceof String && ((String)value).length() == 1) { c = ((String)value).charAt(0); } + if (c == null) return null; if (c == 'Y') return true; if (c == 'N') return false; - throw new MetaException("Cannot extrace boolean from column value " + value); + throw new MetaException("Cannot extract boolean from column value " + value); } private int extractSqlInt(Object field) { @@ -621,7 +630,7 @@ private static String trimCommaList(StringBuilder sb) { } private abstract class ApplyFunc { - public abstract void apply(Target t, Object[] fields); + public abstract void apply(Target t, Object[] fields) throws MetaException; } /** @@ -655,7 +664,7 @@ private static String trimCommaList(StringBuilder sb) { if (fields == null) { fields = iter.next(); } - long nestedId = (Long)fields[keyIndex]; + long nestedId = extractSqlLong(fields[keyIndex]); if (nestedId < id) throw new MetaException("Found entries for unknown ID " + nestedId); if (nestedId > id) break; // fields belong to one of the next entries func.apply(entry.getValue(), fields); @@ -943,7 +952,7 @@ public ColumnStatistics getTableStats( + "\"MAX_COL_LEN\", \"NUM_TRUES\", \"NUM_FALSES\", \"LAST_ANALYZED\" "; private ColumnStatistics makeColumnStats( - List list, ColumnStatisticsDesc csd, int offset) { + List list, ColumnStatisticsDesc csd, int offset) throws MetaException { ColumnStatistics result = new ColumnStatistics(); result.setStatsDesc(csd); List csos = new ArrayList(list.size()); @@ -951,8 +960,8 @@ private ColumnStatistics makeColumnStats( // LastAnalyzed is stored per column but thrift has it per several; // get the lowest for now as nobody actually uses this field. Object laObj = row[offset + 14]; - if (laObj != null && (!csd.isSetLastAnalyzed() || csd.getLastAnalyzed() > (Long)laObj)) { - csd.setLastAnalyzed((Long)laObj); + if (laObj != null && (!csd.isSetLastAnalyzed() || csd.getLastAnalyzed() > extractSqlLong(laObj))) { + csd.setLastAnalyzed(extractSqlLong(laObj)); } ColumnStatisticsData data = new ColumnStatisticsData(); // see STATS_COLLIST diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 6a0eabe..3ea87a0 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -133,6 +133,7 @@ import org.datanucleus.store.rdbms.exceptions.MissingTableException; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; /** @@ -946,6 +947,16 @@ private MTable getMTable(String db, String table) { return tables; } + /** Makes shallow copy of a list to avoid DataNucleus mucking with our objects. */ + private List convertList(List dnList) { + return (dnList == null) ? null : Lists.newArrayList(dnList); + } + + /** Makes shallow copy of a map to avoid DataNucleus mucking with our objects. */ + private Map convertMap(Map dnMap) { + return (dnMap == null) ? null : Maps.newHashMap(dnMap); + } + private Table convertToTable(MTable mtbl) throws MetaException { if (mtbl == null) { return null; @@ -964,9 +975,8 @@ private Table convertToTable(MTable mtbl) throws MetaException { return new Table(mtbl.getTableName(), mtbl.getDatabase().getName(), mtbl .getOwner(), mtbl.getCreateTime(), mtbl.getLastAccessTime(), mtbl .getRetention(), convertToStorageDescriptor(mtbl.getSd()), - convertToFieldSchemas(mtbl.getPartitionKeys()), mtbl.getParameters(), - mtbl.getViewOriginalText(), mtbl.getViewExpandedText(), - tableType); + convertToFieldSchemas(mtbl.getPartitionKeys()), convertMap(mtbl.getParameters()), + mtbl.getViewOriginalText(), mtbl.getViewExpandedText(), tableType); } private MTable convertToMTable(Table tbl) throws InvalidObjectException, @@ -1057,8 +1067,7 @@ private SerDeInfo converToSerDeInfo(MSerDeInfo ms) throws MetaException { if (ms == null) { throw new MetaException("Invalid SerDeInfo object"); } - return new SerDeInfo(ms.getName(), ms.getSerializationLib(), ms - .getParameters()); + return new SerDeInfo(ms.getName(), ms.getSerializationLib(), convertMap(ms.getParameters())); } private MSerDeInfo converToMSerDeInfo(SerDeInfo ms) throws MetaException { @@ -1094,9 +1103,9 @@ private StorageDescriptor convertToStorageDescriptor(MStorageDescriptor msd, StorageDescriptor sd = new StorageDescriptor(noFS ? null : convertToFieldSchemas(mFieldSchemas), msd.getLocation(), msd.getInputFormat(), msd.getOutputFormat(), msd .isCompressed(), msd.getNumBuckets(), converToSerDeInfo(msd - .getSerDeInfo()), msd.getBucketCols(), convertToOrders(msd - .getSortCols()), msd.getParameters()); - SkewedInfo skewedInfo = new SkewedInfo(msd.getSkewedColNames(), + .getSerDeInfo()), convertList(msd.getBucketCols()), convertToOrders(msd + .getSortCols()), convertMap(msd.getParameters())); + SkewedInfo skewedInfo = new SkewedInfo(convertList(msd.getSkewedColNames()), convertToSkewedValues(msd.getSkewedColValues()), covertToSkewedMap(msd.getSkewedColValueLocationMaps())); sd.setSkewedInfo(skewedInfo); @@ -1415,10 +1424,10 @@ private Partition convertToPart(MPartition mpart) throws MetaException { if (mpart == null) { return null; } - return new Partition(mpart.getValues(), mpart.getTable().getDatabase() + return new Partition(convertList(mpart.getValues()), mpart.getTable().getDatabase() .getName(), mpart.getTable().getTableName(), mpart.getCreateTime(), mpart.getLastAccessTime(), convertToStorageDescriptor(mpart.getSd()), - mpart.getParameters()); + convertMap(mpart.getParameters())); } private Partition convertToPart(String dbName, String tblName, MPartition mpart) @@ -1426,9 +1435,9 @@ private Partition convertToPart(String dbName, String tblName, MPartition mpart) if (mpart == null) { return null; } - return new Partition(mpart.getValues(), dbName, tblName, mpart.getCreateTime(), - mpart.getLastAccessTime(), convertToStorageDescriptor(mpart.getSd(), false), - mpart.getParameters()); + return new Partition(convertList(mpart.getValues()), dbName, tblName, + mpart.getCreateTime(), mpart.getLastAccessTime(), + convertToStorageDescriptor(mpart.getSd(), false), convertMap(mpart.getParameters())); } @Override diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java b/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java index f731dab..a4f9b0a 100755 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java @@ -570,7 +570,15 @@ private static int calculateListBucketingDMLDepth(StorageDescriptor desc) { public static String makePartName(List partCols, List vals, String defaultStr) throws MetaException { if ((partCols.size() != vals.size()) || (partCols.size() == 0)) { - throw new MetaException("Invalid partition key & values"); + String errorStr = "Invalid partition key & values; keys ["; + for (FieldSchema fs : partCols) { + errorStr += (fs.getName() + ", "); + } + errorStr += "], values ["; + for (String val : vals) { + errorStr += (val + ", "); + } + throw new MetaException(errorStr + "]"); } List colNames = new ArrayList(); for (FieldSchema col: partCols) { diff --git a/ql/src/test/results/clientnegative/unset_table_property.q.out b/ql/src/test/results/clientnegative/unset_table_property.q.out index fe69b6f..6d7d34c 100644 --- a/ql/src/test/results/clientnegative/unset_table_property.q.out +++ b/ql/src/test/results/clientnegative/unset_table_property.q.out @@ -22,10 +22,10 @@ numFiles 0 c 3 #### A masked pattern was here #### a 1 -COLUMN_STATS_ACCURATE false #### A masked pattern was here #### -numRows -1 +COLUMN_STATS_ACCURATE false totalSize 0 +numRows -1 rawDataSize -1 FAILED: SemanticException [Error 10215]: Please use the following syntax if not sure whether the property existed or not: ALTER TABLE tableName UNSET TBLPROPERTIES IF EXISTS (key1, key2, ...) diff --git a/ql/src/test/results/clientpositive/nullformatCTAS.q.out b/ql/src/test/results/clientpositive/nullformatCTAS.q.out index a1c0c3d..85eaf37 100644 --- a/ql/src/test/results/clientpositive/nullformatCTAS.q.out +++ b/ql/src/test/results/clientpositive/nullformatCTAS.q.out @@ -168,10 +168,10 @@ LOCATION #### A masked pattern was here #### TBLPROPERTIES ( 'numFiles'='1', - 'COLUMN_STATS_ACCURATE'='true', #### A masked pattern was here #### - 'numRows'='10', + 'COLUMN_STATS_ACCURATE'='true', 'totalSize'='80', + 'numRows'='10', 'rawDataSize'='70') 1.01 1.01 diff --git a/ql/src/test/results/clientpositive/show_create_table_alter.q.out b/ql/src/test/results/clientpositive/show_create_table_alter.q.out index 64eb114..c4baa5b 100644 --- a/ql/src/test/results/clientpositive/show_create_table_alter.q.out +++ b/ql/src/test/results/clientpositive/show_create_table_alter.q.out @@ -69,13 +69,12 @@ OUTPUTFORMAT LOCATION #### A masked pattern was here #### TBLPROPERTIES ( - 'EXTERNAL'='FALSE', 'numFiles'='0', + 'EXTERNAL'='FALSE', #### A masked pattern was here #### 'COLUMN_STATS_ACCURATE'='false', -#### A masked pattern was here #### - 'numRows'='-1', 'totalSize'='0', + 'numRows'='-1', 'rawDataSize'='-1') PREHOOK: query: -- Alter the table comment, change the EXTERNAL property back and test SHOW CREATE TABLE on the change. ALTER TABLE tmp_showcrt1 SET TBLPROPERTIES ('comment'='changed comment', 'EXTERNAL'='TRUE') @@ -114,9 +113,8 @@ TBLPROPERTIES ( 'numFiles'='0', #### A masked pattern was here #### 'COLUMN_STATS_ACCURATE'='false', -#### A masked pattern was here #### - 'numRows'='-1', 'totalSize'='0', + 'numRows'='-1', 'rawDataSize'='-1') PREHOOK: query: -- Change the 'SORTBUCKETCOLSPREFIX' property and test SHOW CREATE TABLE. The output should not change. ALTER TABLE tmp_showcrt1 SET TBLPROPERTIES ('SORTBUCKETCOLSPREFIX'='FALSE') @@ -155,9 +153,8 @@ TBLPROPERTIES ( 'numFiles'='0', #### A masked pattern was here #### 'COLUMN_STATS_ACCURATE'='false', -#### A masked pattern was here #### - 'numRows'='-1', 'totalSize'='0', + 'numRows'='-1', 'rawDataSize'='-1') PREHOOK: query: -- Alter the storage handler of the table, and test SHOW CREATE TABLE. ALTER TABLE tmp_showcrt1 SET TBLPROPERTIES ('storage_handler'='org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler') @@ -196,9 +193,8 @@ TBLPROPERTIES ( 'numFiles'='0', #### A masked pattern was here #### 'COLUMN_STATS_ACCURATE'='false', -#### A masked pattern was here #### - 'numRows'='-1', 'totalSize'='0', + 'numRows'='-1', 'rawDataSize'='-1') PREHOOK: query: DROP TABLE tmp_showcrt1 PREHOOK: type: DROPTABLE diff --git a/ql/src/test/results/clientpositive/show_tblproperties.q.out b/ql/src/test/results/clientpositive/show_tblproperties.q.out index 8ca4683..c60e710 100644 --- a/ql/src/test/results/clientpositive/show_tblproperties.q.out +++ b/ql/src/test/results/clientpositive/show_tblproperties.q.out @@ -33,11 +33,11 @@ POSTHOOK: type: SHOW_TBLPROPERTIES numFiles 0 #### A masked pattern was here #### tmp true -COLUMN_STATS_ACCURATE false #### A masked pattern was here #### -bar bar value -numRows -1 +COLUMN_STATS_ACCURATE false totalSize 0 +numRows -1 +bar bar value rawDataSize -1 PREHOOK: query: show tblproperties tmpfoo("bar") PREHOOK: type: SHOW_TBLPROPERTIES diff --git a/ql/src/test/results/clientpositive/unset_table_view_property.q.out b/ql/src/test/results/clientpositive/unset_table_view_property.q.out index aafc206..f4f1e91 100644 --- a/ql/src/test/results/clientpositive/unset_table_view_property.q.out +++ b/ql/src/test/results/clientpositive/unset_table_view_property.q.out @@ -29,10 +29,10 @@ numFiles 0 c 3 #### A masked pattern was here #### a 1 -COLUMN_STATS_ACCURATE false #### A masked pattern was here #### -numRows -1 +COLUMN_STATS_ACCURATE false totalSize 0 +numRows -1 rawDataSize -1 PREHOOK: query: -- UNSET all the properties ALTER TABLE testTable UNSET TBLPROPERTIES ('a', 'c') @@ -51,9 +51,8 @@ POSTHOOK: type: SHOW_TBLPROPERTIES numFiles 0 #### A masked pattern was here #### COLUMN_STATS_ACCURATE false -#### A masked pattern was here #### -numRows -1 totalSize 0 +numRows -1 rawDataSize -1 PREHOOK: query: ALTER TABLE testTable SET TBLPROPERTIES ('a'='1', 'c'='3', 'd'='4') PREHOOK: type: ALTERTABLE_PROPERTIES @@ -67,16 +66,16 @@ PREHOOK: query: SHOW TBLPROPERTIES testTable PREHOOK: type: SHOW_TBLPROPERTIES POSTHOOK: query: SHOW TBLPROPERTIES testTable POSTHOOK: type: SHOW_TBLPROPERTIES -d 4 numFiles 0 +d 4 #### A masked pattern was here #### c 3 #### A masked pattern was here #### a 1 -COLUMN_STATS_ACCURATE false #### A masked pattern was here #### -numRows -1 +COLUMN_STATS_ACCURATE false totalSize 0 +numRows -1 rawDataSize -1 PREHOOK: query: -- UNSET a subset of the properties ALTER TABLE testTable UNSET TBLPROPERTIES ('a', 'd') @@ -97,9 +96,8 @@ numFiles 0 c 3 #### A masked pattern was here #### COLUMN_STATS_ACCURATE false -#### A masked pattern was here #### -numRows -1 totalSize 0 +numRows -1 rawDataSize -1 PREHOOK: query: -- the same property being UNSET multiple times ALTER TABLE testTable UNSET TBLPROPERTIES ('c', 'c', 'c') @@ -118,9 +116,8 @@ POSTHOOK: type: SHOW_TBLPROPERTIES numFiles 0 #### A masked pattern was here #### COLUMN_STATS_ACCURATE false -#### A masked pattern was here #### -numRows -1 totalSize 0 +numRows -1 rawDataSize -1 PREHOOK: query: ALTER TABLE testTable SET TBLPROPERTIES ('a'='1', 'b' = '2', 'c'='3', 'd'='4') PREHOOK: type: ALTERTABLE_PROPERTIES @@ -134,17 +131,17 @@ PREHOOK: query: SHOW TBLPROPERTIES testTable PREHOOK: type: SHOW_TBLPROPERTIES POSTHOOK: query: SHOW TBLPROPERTIES testTable POSTHOOK: type: SHOW_TBLPROPERTIES -d 4 numFiles 0 +d 4 #### A masked pattern was here #### b 2 c 3 #### A masked pattern was here #### a 1 -COLUMN_STATS_ACCURATE false #### A masked pattern was here #### -numRows -1 +COLUMN_STATS_ACCURATE false totalSize 0 +numRows -1 rawDataSize -1 PREHOOK: query: -- UNSET a subset of the properties and some non-existed properties using IF EXISTS ALTER TABLE testTable UNSET TBLPROPERTIES IF EXISTS ('b', 'd', 'b', 'f') @@ -165,10 +162,10 @@ numFiles 0 c 3 #### A masked pattern was here #### a 1 -COLUMN_STATS_ACCURATE false #### A masked pattern was here #### -numRows -1 +COLUMN_STATS_ACCURATE false totalSize 0 +numRows -1 rawDataSize -1 PREHOOK: query: -- UNSET a subset of the properties and some non-existed properties using IF EXISTS ALTER TABLE testTable UNSET TBLPROPERTIES IF EXISTS ('b', 'd', 'c', 'f', 'x', 'y', 'z') @@ -187,10 +184,10 @@ POSTHOOK: type: SHOW_TBLPROPERTIES numFiles 0 #### A masked pattern was here #### a 1 -COLUMN_STATS_ACCURATE false #### A masked pattern was here #### -numRows -1 +COLUMN_STATS_ACCURATE false totalSize 0 +numRows -1 rawDataSize -1 PREHOOK: query: -- UNSET VIEW PROPERTIES CREATE VIEW testView AS SELECT value FROM src WHERE key=86