diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 4601e09..11e9ddd 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -5794,6 +5794,9 @@ public int get_num_partitions_by_filter(final String dbName, final String tblName, final String filter) throws TException { String[] parsedDbName = parseDbName(dbName, conf); + if (parsedDbName[DB_NAME] == null || tblName == null) { + throw new MetaException("The DB and table name cannot be null."); + } startTableFunction("get_num_partitions_by_filter", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName); diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 6645e55..8688170 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -3145,7 +3145,7 @@ private Collection getPartitionPsQueryResults(String catName, String dbName, Str "table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3"); query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); query.setOrdering("partitionName ascending"); - if (max > 0) { + if (max >= 0) { query.setRange(0, max); } mparts = (List) query.execute(tableName, dbName, catName); diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java index d608e50..32267b4 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java @@ -361,7 +361,8 @@ private void generateJDOFilterOverPartitions(Configuration conf, Table table, boolean isOpEquals = operator == Operator.EQUALS; if (isOpEquals || operator == Operator.NOTEQUALS || operator == Operator.NOTEQUALS2) { - makeFilterForEquals(keyName, valueAsString, paramName, params, + String partitionKey = table.getPartitionKeys().get(partitionColumnIndex).getName(); + makeFilterForEquals(partitionKey, valueAsString, paramName, params, partitionColumnIndex, partitionColumnCount, isOpEquals, filterBuilder); return; } diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestListPartitions.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestListPartitions.java index a8b6e31..e5d1058 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestListPartitions.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestListPartitions.java @@ -52,7 +52,6 @@ import org.junit.After; import org.junit.Assert; -import org.junit.Assume; import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -253,12 +252,8 @@ public void testListPartitionsAll() throws Exception { partitions = client.listPartitions(DB_NAME, TABLE_NAME, (short)1); assertPartitionsHaveCorrectValues(partitions, testValues.subList(0, 1)); - // HIVE-18977 - if (MetastoreConf.getBoolVar(metaStore.getConf(), MetastoreConf.ConfVars.TRY_DIRECT_SQL)) { - partitions = client.listPartitions(DB_NAME, TABLE_NAME, (short) 0); - assertTrue(partitions.isEmpty()); - } - + partitions = client.listPartitions(DB_NAME, TABLE_NAME, (short) 0); + assertTrue(partitions.isEmpty()); } @Test(expected = MetaException.class) @@ -270,14 +265,14 @@ public void testListPartitionsAllHighMaxParts() throws Exception { @Test public void testListPartitionsAllNoParts() throws Exception { - Table t = createTestTable(client, DB_NAME, TABLE_NAME, Lists.newArrayList("yyyy", "mm", "dd")); + createTestTable(client, DB_NAME, TABLE_NAME, Lists.newArrayList("yyyy", "mm", "dd")); List partitions = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1); assertTrue(partitions.isEmpty()); } @Test(expected = NoSuchObjectException.class) public void testListPartitionsAllNoTable() throws Exception { - List partitions = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1); + client.listPartitions(DB_NAME, TABLE_NAME, (short)-1); } @Test(expected = NoSuchObjectException.class) @@ -302,8 +297,7 @@ public void testListPartitionsAllNoTblName() throws Exception { public void testListPartitionsAllNullTblName() throws Exception { try { createTable3PartCols1Part(client); - List partitions = client.listPartitions(DB_NAME, - (String)null, (short)-1); + client.listPartitions(DB_NAME, (String)null, (short)-1); fail("Should have thrown exception"); } catch (NullPointerException | TTransportException e) { //TODO: should not throw different exceptions for different HMS deployment types @@ -416,11 +410,8 @@ public void testListPartitionSpecs() throws Exception { partSpecProxy = client.listPartitionSpecs(DB_NAME, TABLE_NAME, 2); assertPartitionsSpecProxy(partSpecProxy, testValues.subList(0, 2)); - // HIVE-18977 - if (MetastoreConf.getBoolVar(metaStore.getConf(), MetastoreConf.ConfVars.TRY_DIRECT_SQL)) { - partSpecProxy = client.listPartitionSpecs(DB_NAME, TABLE_NAME, 0); - assertPartitionsSpecProxy(partSpecProxy, testValues.subList(0, 0)); - } + partSpecProxy = client.listPartitionSpecs(DB_NAME, TABLE_NAME, 0); + assertPartitionsSpecProxy(partSpecProxy, testValues.subList(0, 0)); } @Test(expected = NoSuchObjectException.class) @@ -745,18 +736,88 @@ public void testListPartitionsByFilter() throws Exception { "yyyy=\"2017\" OR " + "mm=\"02\"", (short)0); assertTrue(partitions.isEmpty()); - // HIVE-18977 - if (MetastoreConf.getBoolVar(metaStore.getConf(), MetastoreConf.ConfVars.TRY_DIRECT_SQL)) { - partitions = client.listPartitionsByFilter(DB_NAME, TABLE_NAME, - "yYyY=\"2017\"", (short) -1); - assertPartitionsHaveCorrectValues(partitions, partValues.subList(2, 4)); - } - partitions = client.listPartitionsByFilter(DB_NAME, TABLE_NAME, "yyyy=\"2017\" AND mm=\"99\"", (short)-1); assertTrue(partitions.isEmpty()); } + @Test + public void testListPartitionsByFilterCaseInsensitive() throws Exception { + String tableName = TABLE_NAME + "_caseinsensitive"; + Table t = createTestTable(client, DB_NAME, tableName, + Lists.newArrayList("yyyy", "month", "day"), false); + List> testValues = Lists.newArrayList( + Lists.newArrayList("2017", "march", "11"), + Lists.newArrayList("2017", "march", "15"), + Lists.newArrayList("2017", "may", "15"), + Lists.newArrayList("2018", "march", "11"), + Lists.newArrayList("2018", "september", "7")); + + for(List vals : testValues) { + addPartition(client, t, vals); + } + + List partitions = client.listPartitionsByFilter(DB_NAME, tableName, + "yYyY=\"2017\"", (short) -1); + assertPartitionsHaveCorrectValues(partitions, testValues.subList(0, 3)); + + partitions = client.listPartitionsByFilter(DB_NAME, tableName, + "yYyY=\"2017\" AND mOnTh=\"may\"", (short) -1); + assertPartitionsHaveCorrectValues(partitions, testValues.subList(2, 3)); + + partitions = client.listPartitionsByFilter(DB_NAME, tableName, + "yYyY!=\"2017\"", (short) -1); + assertPartitionsHaveCorrectValues(partitions, testValues.subList(3, 5)); + + partitions = client.listPartitionsByFilter(DB_NAME, tableName, + "mOnTh=\"september\"", (short) -1); + assertPartitionsHaveCorrectValues(partitions, testValues.subList(4, 5)); + + partitions = client.listPartitionsByFilter(DB_NAME, tableName, + "mOnTh like \"m.*\"", (short) -1); + assertPartitionsHaveCorrectValues(partitions, testValues.subList(0, 4)); + + partitions = client.listPartitionsByFilter(DB_NAME, tableName, + "yYyY=\"2018\" AND mOnTh like \"m.*\"", (short) -1); + assertPartitionsHaveCorrectValues(partitions, testValues.subList(3, 4)); + client.dropTable(DB_NAME, tableName); + } + + @Test + public void testListPartitionsByFilterCaseSensitive() throws Exception { + String tableName = TABLE_NAME + "_casesensitive"; + Table t = createTestTable(client, DB_NAME, tableName, + Lists.newArrayList("yyyy", "month", "day"), false); + List> testValues = Lists.newArrayList( + Lists.newArrayList("2017", "march", "11"), + Lists.newArrayList("2017", "march", "15"), + Lists.newArrayList("2017", "may", "15"), + Lists.newArrayList("2018", "march", "11"), + Lists.newArrayList("2018", "april", "7")); + + for(List vals : testValues) { + addPartition(client, t, vals); + } + + List partitions = client.listPartitionsByFilter(DB_NAME, tableName, + "month=\"mArCh\"", (short) -1); + Assert.assertTrue(partitions.isEmpty()); + + partitions = client.listPartitionsByFilter(DB_NAME, tableName, + "yyyy=\"2017\" AND month=\"May\"", (short) -1); + Assert.assertTrue(partitions.isEmpty()); + + partitions = client.listPartitionsByFilter(DB_NAME, tableName, + "yyyy=\"2017\" AND month!=\"mArCh\"", (short) -1); + assertPartitionsHaveCorrectValues(partitions, testValues.subList(0, 3)); + + partitions = client.listPartitionsByFilter(DB_NAME, tableName, + "month like \"M.*\"", (short) -1); + Assert.assertTrue(partitions.isEmpty()); + client.dropTable(DB_NAME, tableName); + + } + @Test(expected = MetaException.class) public void testListPartitionsByFilterInvalidFilter() throws Exception { createTable4PartColsParts(client); @@ -987,15 +1048,10 @@ public void testGetNumPartitionsByFilterNoDb() throws Exception { client.getNumPartitionsByFilter(DB_NAME, TABLE_NAME, "yyyy=\"2017\""); } - @Test + @Test(expected = MetaException.class) public void testGetNumPartitionsByFilterNullTblName() throws Exception { - try { - createTable4PartColsParts(client); - client.getNumPartitionsByFilter(DB_NAME, null, "yyyy=\"2017\""); - fail("Should have thrown exception"); - } catch (MetaException | TTransportException e) { - //TODO: should not throw different exceptions for different HMS deployment types - } + createTable4PartColsParts(client); + client.getNumPartitionsByFilter(DB_NAME, null, "yyyy=\"2017\""); } @Test(expected = MetaException.class)