diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 7b3acad511..142e10878e 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -5766,7 +5766,7 @@ private void initialize(Class cls) { // set the hardcoded value first, so anything in hiveserver2-site.xml can override it set(ConfVars.METASTORE_CLIENT_CAPABILITIES.varname, "EXTWRITE,EXTREAD,HIVEBUCKET2,HIVEFULLACIDREAD," + "HIVEFULLACIDWRITE,HIVECACHEINVALIDATE,HIVEMANAGESTATS,HIVEMANAGEDINSERTWRITE,HIVEMANAGEDINSERTREAD," - + "HIVESQL,HIVEMQT,HIVEONLYMQTWRITE"); + + "HIVESQL,HIVEMQT,HIVEONLYMQTWRITE,ACCEPTS_UNMODIFIED_METADATA"); if (hiveServer2SiteUrl != null) { addResource(hiveServer2SiteUrl); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java index eba6610c64..27a1557138 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java @@ -193,7 +193,14 @@ public void testTransformerExternalTable() throws Exception { tbl2 = client.getTable(dbName, tblName); assertEquals("Expected buckets does not match:", buckets, tbl2.getSd().getNumBuckets()); // no transformation assertEquals("Table access type does not match expected value:" + tblName, - 8, tbl2.getAccessType()); // RW with HIVEBUCKET2 but no EXTWRITE + ACCESSTYPE_READWRITE, tbl2.getAccessType()); // RW with HIVEBUCKET2 but no EXTWRITE + resetHMSClient(); + + setHMSClient("testTransformerAcceptsUnmodifiedMetadata", (new String[] { "ACCEPTS_UNMODIFIED_METADATA" })); + tbl2 = client.getTable(dbName, tblName); + assertEquals("Expected buckets does not match:", buckets, tbl2.getSd().getNumBuckets()); // no transformation + assertEquals("Table access type does not match expected value:" + tblName, + ACCESSTYPE_READONLY, tbl2.getAccessType()); // RO without HIVEBUCKET2 but with ACCEPTS_UNMODIFIED_METADATA resetHMSClient(); tblName = "test_ext_bucketed_wc"; @@ -217,7 +224,6 @@ public void testTransformerExternalTable() throws Exception { assertEquals(buckets, tbl2.getSd().getNumBuckets()); // client has the HIVEBUCKET2 capability, retain bucketing info assertNull(tbl2.getRequiredWriteCapabilities()); assertNull(tbl2.getRequiredReadCapabilities()); - resetHMSClient(); setHMSClient("testTransformerExternalTableRO", (new String[] { "EXTREAD", "EXTWRITE"})); @@ -229,7 +235,17 @@ public void testTransformerExternalTable() throws Exception { tbl2.getRequiredWriteCapabilities()); assertTrue("Returned required capabilities list does not contain HIVEBUCKET2", tbl2.getRequiredWriteCapabilities().contains("HIVEBUCKET2")); + resetHMSClient(); + setHMSClient("testTransformerExternalTableROwAUM", (new String[] { "EXTREAD", "EXTWRITE", "ACCEPTS_UNMODIFIED_METADATA"})); + tbl2 = client.getTable(dbName, tblName); + assertEquals("Table access type does not match the expected value:" + tblName, + ACCESSTYPE_READONLY, tbl2.getAccessType()); + assertEquals(buckets, tbl2.getSd().getNumBuckets()); // client has no HIVEBUCKET2 capability, but has ACCEPTS_UNMODIFIED_METADATA + assertNotNull("Required write capabilities is null", + tbl2.getRequiredWriteCapabilities()); + assertTrue("Returned required capabilities list does not contain HIVEBUCKET2", + tbl2.getRequiredWriteCapabilities().contains("HIVEBUCKET2")); resetHMSClient(); tblName = "test_ext_unbucketed_wc"; @@ -286,7 +302,6 @@ public void testTransformerExternalTable() throws Exception { assertEquals("Table access type does not match the expected value:" + tblName, ACCESSTYPE_READWRITE, tbl2.getAccessType()); assertEquals(buckets, tbl2.getSd().getNumBuckets()); // client has SPARKBUCKET capability - resetHMSClient(); LOG.info("Test execution complete:testTransformerExternalTable"); @@ -867,7 +882,17 @@ public void testGetPartitionsByNames() throws Exception { parts = client.getPartitionsByNames(dbName, tblName, partValues, false, null); for (Partition part : parts) { - assertEquals("Partition bucket count does not match", -1, part.getSd().getNumBuckets()); + assertEquals("Partition bucket count does not match", bucketCount, part.getSd().getNumBuckets()); + } + + // processor has ACCEPTS_UNMODIFIED_METADATA + capabilities.clear(); + capabilities.add("ACCEPTS_UNMODIFIED_METADATA"); + setHMSClient("TestGetPartitionByNames#3", (String[])(capabilities.toArray(new String[0]))); + parts = client.getPartitionsByNames(dbName, tblName, partValues, false, null); + + for (Partition part : parts) { + assertEquals("Partition bucket count does not match", bucketCount, part.getSd().getNumBuckets()); } tblName = "test_parts_mgd_insert_wc"; @@ -1860,7 +1885,7 @@ private void addPartition(IMetaStoreClient client, Table table, List val PartitionBuilder partitionBuilder = new PartitionBuilder().inTable(table); values.forEach(val -> partitionBuilder.addValue(val)); Partition p = partitionBuilder.build(conf); - p.getSd().setNumBuckets(-1); // PartitionBuilder uses 0 as default whereas we use -1 for Tables. + p.getSd().setNumBuckets(table.getSd().getNumBuckets()); client.add_partition(p); } } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java index fa16192e6e..3c483eebdd 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java @@ -65,6 +65,7 @@ private static final String HIVESQL = "HIVESQL".intern(); private static final String OBJCAPABILITIES = "OBJCAPABILITIES".intern(); private static final String MANAGERAWMETADATA = "MANAGE_RAW_METADATA".intern(); + private static final String ACCEPTSUNMODIFIEDMETADATA = "ACCEPTS_UNMODIFIED_METADATA".intern(); private static final List ACIDCOMMONWRITELIST = new ArrayList(Arrays.asList( HIVEMANAGESTATS, @@ -137,10 +138,12 @@ public MetastoreDefaultTransformer(IHMSHandler handler) throws HiveMetaException newTable.setAccessType(ACCESSTYPE_READONLY); requiredWrites.add(HIVEBUCKET2); StorageDescriptor newSd = new StorageDescriptor(table.getSd()); - newSd.setNumBuckets(-1); // remove bucketing info + if (!processorCapabilities.contains(ACCEPTSUNMODIFIEDMETADATA)) { + LOG.debug("Bucketed table without HIVEBUCKET2 capability, removed bucketing info from table"); + newSd.setNumBuckets(-1); // remove bucketing info + } newTable.setSd(newSd); newTable.setRequiredWriteCapabilities(requiredWrites); - LOG.info("Bucketed table without HIVEBUCKET2 capability, removed bucketing info from table"); } } else { // Unbucketed if (processorCapabilities.contains(EXTWRITE) && processorCapabilities.contains(EXTREAD)) { @@ -269,21 +272,20 @@ public MetastoreDefaultTransformer(IHMSHandler handler) throws HiveMetaException } Table newTable = new Table(table); - boolean removedBucketing = false; - if (requiredCapabilities.contains(HIVEBUCKET2) && !processorCapabilities.contains(HIVEBUCKET2)) { StorageDescriptor newSd = new StorageDescriptor(table.getSd()); - newSd.setNumBuckets(-1); // removing bucketing if HIVEBUCKET2 isnt specified + if (!processorCapabilities.contains(ACCEPTSUNMODIFIEDMETADATA)) { + newSd.setNumBuckets(-1); // removing bucketing if HIVEBUCKET2 isnt specified + LOG.debug("Bucketed table without HIVEBUCKET2 capability, removed bucketing info from table"); + } newTable.setSd(newSd); - removedBucketing = true; newTable.setAccessType(ACCESSTYPE_READONLY); LOG.debug("Adding HIVEBUCKET2 to requiredWrites"); requiredWrites.add(HIVEBUCKET2); - LOG.info("Removed bucketing information from table"); } if (requiredCapabilities.contains(EXTWRITE) && processorCapabilities.contains(EXTWRITE)) { - if (!removedBucketing) { + if (!isBucketed) { LOG.info("EXTWRITE Matches, accessType=" + ACCESSTYPE_READWRITE); newTable.setAccessType(ACCESSTYPE_READWRITE); ret.put(newTable, requiredCapabilities); @@ -468,7 +470,7 @@ public MetastoreDefaultTransformer(IHMSHandler handler) throws HiveMetaException String tCapabilities = params.get(OBJCAPABILITIES); if (partition.getSd() != null) { partBuckets = partition.getSd().getNumBuckets(); - LOG.info("Number of original part buckets=" + partBuckets); + LOG.debug("Number of original part buckets=" + partBuckets); } else { partBuckets = 0; } @@ -481,7 +483,8 @@ public MetastoreDefaultTransformer(IHMSHandler handler) throws HiveMetaException if (partBuckets > 0 && !processorCapabilities.contains(HIVEBUCKET2)) { Partition newPartition = new Partition(partition); StorageDescriptor newSd = new StorageDescriptor(partition.getSd()); - newSd.setNumBuckets(-1); // remove bucketing info + if (!processorCapabilities.contains(ACCEPTSUNMODIFIEDMETADATA)) + newSd.setNumBuckets(-1); // remove bucketing info newPartition.setSd(newSd); ret.add(newPartition); } else { @@ -494,7 +497,8 @@ public MetastoreDefaultTransformer(IHMSHandler handler) throws HiveMetaException if (partBuckets > 0 && !processorCapabilities.contains(HIVEBUCKET2)) { Partition newPartition = new Partition(partition); StorageDescriptor newSd = new StorageDescriptor(partition.getSd()); - newSd.setNumBuckets(-1); // remove bucketing info + if (!processorCapabilities.contains(ACCEPTSUNMODIFIEDMETADATA)) + newSd.setNumBuckets(-1); // remove bucketing info newPartition.setSd(newSd); ret.add(newPartition); break; @@ -520,7 +524,8 @@ public MetastoreDefaultTransformer(IHMSHandler handler) throws HiveMetaException if (requiredCapabilities.contains(HIVEBUCKET2) && !processorCapabilities.contains(HIVEBUCKET2)) { Partition newPartition = new Partition(partition); StorageDescriptor newSd = new StorageDescriptor(partition.getSd()); - newSd.setNumBuckets(-1); // removing bucketing if HIVEBUCKET2 isnt specified + if (!processorCapabilities.contains(ACCEPTSUNMODIFIEDMETADATA)) + newSd.setNumBuckets(-1); // removing bucketing if HIVEBUCKET2 isnt specified newPartition.setSd(newSd); LOG.info("Removed bucketing information from partition"); ret.add(newPartition); @@ -532,7 +537,8 @@ public MetastoreDefaultTransformer(IHMSHandler handler) throws HiveMetaException if (!processorCapabilities.contains(HIVEBUCKET2)) { Partition newPartition = new Partition(partition); StorageDescriptor newSd = new StorageDescriptor(partition.getSd()); - newSd.setNumBuckets(-1); // remove bucketing info + if (!processorCapabilities.contains(ACCEPTSUNMODIFIEDMETADATA)) + newSd.setNumBuckets(-1); // remove bucketing info newPartition.setSd(newSd); ret.add(newPartition); break;