commit 608d1a0fe813d7ccdee6d2f44ed971c70b14a0a1 Author: Bharath Krishna Date: Tue Aug 14 12:38:55 2018 -0700 HIVE-19254 : NumberFormatException in MetaStoreUtils.isFastStatsSame diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java index 69846b77d1..23ac7da98f 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java @@ -369,9 +369,9 @@ static boolean containsAllFastStats(Map partParams) { public static boolean isFastStatsSame(Partition oldPart, Partition newPart) { // requires to calculate stats if new and old have different fast stats - if ((oldPart != null) && (oldPart.getParameters() != null)) { + if ((oldPart != null) && oldPart.isSetParameters() && newPart != null && newPart.isSetParameters()) { for (String stat : StatsSetupConst.FAST_STATS) { - if (oldPart.getParameters().containsKey(stat)) { + if (oldPart.getParameters().containsKey(stat) && newPart.getParameters().containsKey(stat)) { Long oldStat = Long.parseLong(oldPart.getParameters().get(stat)); Long newStat = Long.parseLong(newPart.getParameters().get(stat)); if (!oldStat.equals(newStat)) { diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreUtils.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreUtils.java index d09ac8ced9..ffe94492b8 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreUtils.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreUtils.java @@ -29,13 +29,17 @@ import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; +import org.apache.hadoop.hive.metastore.client.builder.PartitionBuilder; import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; import org.apache.thrift.TException; import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -43,14 +47,18 @@ import java.util.Map; import static org.apache.hadoop.hive.common.StatsSetupConst.COLUMN_STATS_ACCURATE; +import static org.apache.hadoop.hive.common.StatsSetupConst.FAST_STATS; import static org.apache.hadoop.hive.common.StatsSetupConst.NUM_FILES; import static org.apache.hadoop.hive.common.StatsSetupConst.NUM_ERASURE_CODED_FILES; import static org.apache.hadoop.hive.common.StatsSetupConst.STATS_GENERATED; import static org.apache.hadoop.hive.common.StatsSetupConst.TOTAL_SIZE; import static org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.updateTableStatsSlow; +import static org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.isFastStatsSame; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -277,6 +285,63 @@ public void testUpdateTableStatsSlow_doesNotUpdateStats() throws TException { verify(wh, never()).getFileStatusesForUnpartitionedTable(db, tbl2); } + @Test + public void testIsFastStatsSameHandleNull() throws TException { + + FieldSchema fs = new FieldSchema("date", "string", "date column"); + List cols = Collections.singletonList(fs); + + Table tbl = new TableBuilder() + .setDbName(DB_NAME) + .setTableName(TABLE_NAME) + .addCol("id", "int") + .build(null); + List vals = new ArrayList(); + vals.add("val1"); + Partition oldPart; + Partition newPart; + + PartitionBuilder partitionBuilder = new PartitionBuilder().inTable(tbl); + vals.forEach(val -> partitionBuilder.addValue(val)); + + oldPart = partitionBuilder.build(null); + newPart = partitionBuilder.build(null); + + Map oldParams = new HashMap<>(); + Map newParams = new HashMap<>(); + + //Test case where all parameters are present and their values are same + long testVal = 1; + for (String key : FAST_STATS) { + oldParams.put(key, String.valueOf(testVal)); + newParams.put(key, String.valueOf(testVal)); + } + oldPart.setParameters(oldParams); + newPart.setParameters(newParams); + assertTrue(isFastStatsSame(oldPart, newPart)); + + //Test case where all parameters are present and their values are different + for (String key : FAST_STATS) { + oldParams.put(key, String.valueOf(testVal)); + newParams.put(key, String.valueOf(++testVal)); + } + oldPart.setParameters(oldParams); + newPart.setParameters(newParams); + assertFalse(isFastStatsSame(oldPart, newPart)); + + //Test case where newPart is null + assertFalse(isFastStatsSame(oldPart, null)); + + //Test case where oldPart is null + assertFalse(isFastStatsSame(null, newPart)); + + //Test case where one or all of the FAST_STATS parameters are not present in newPart + Map randomParams = new HashMap(); + randomParams.put("randomParam1", "randomVal1"); + newPart.setParameters(randomParams); + assertFalse(isFastStatsSame(oldPart, newPart)); + } + /** * Build a FileStatus object. */