diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java index 7afef0fce606cbe12e8034095ae77161d68742cb..4c01311117e85a62f62d6157c177cf58ef28adfc 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.txn.compactor; import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -31,19 +32,25 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.cli.CliSessionState; +import org.apache.hadoop.hive.common.ValidTxnList; +import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.api.CompactionType; +import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.ShowCompactRequest; import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.txn.CompactionInfo; import org.apache.hadoop.hive.metastore.txn.TxnDbUtil; import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.ql.DriverFactory; import org.apache.hadoop.hive.ql.IDriver; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.HiveInputFormat; +import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hive.streaming.HiveStreamingConnection; import org.apache.hive.streaming.StreamingConnection; @@ -55,6 +62,8 @@ import static org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.executeStatementOnDriver; import static org.apache.hadoop.hive.ql.txn.compactor.CompactorTestUtil.executeStatementOnDriverAndReturnResults; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; @SuppressWarnings("deprecation") public class TestCrudCompactorOnTez { @@ -923,6 +932,44 @@ public void testCompactionWithSchemaEvolutionNoBucketsMultipleReducers() throws executeStatementOnDriver("drop table " + tblName, driver); } + /** + * Tests whether hive.llap.io.etl.skip.format config is handled properly whenever QueryCompactor#runCompactionQueries + * is invoked. + * @throws Exception + */ + @Test + public void testLlapCacheOffDuringCompaction() throws Exception { + // Setup + QueryCompactor qc = new QueryCompactor() { + @Override + void runCompaction(HiveConf hiveConf, Table table, Partition partition, StorageDescriptor storageDescriptor, + ValidWriteIdList writeIds, CompactionInfo compactionInfo) throws IOException { + } + + @Override + protected void commitCompaction(String dest, String tmpTableName, HiveConf conf, ValidWriteIdList actualWriteIds, + long compactorTxnId) throws IOException, HiveException { + } + }; + StorageDescriptor sdMock = mock(StorageDescriptor.class); + doAnswer(invocationOnMock -> { + return null; + }).when(sdMock).getLocation(); + List emptyQueries = new ArrayList<>(); + HiveConf hiveConf = new HiveConf(); + hiveConf.set(ValidTxnList.VALID_TXNS_KEY, "8:9223372036854775807::"); + + // Check for default case. + qc.runCompactionQueries(hiveConf, null, sdMock, null, null, emptyQueries, emptyQueries, emptyQueries); + Assert.assertEquals("all", hiveConf.getVar(HiveConf.ConfVars.LLAP_IO_ETL_SKIP_FORMAT)); + + // Check for case where hive.llap.io.etl.skip.format is explicitly set to none - as to always use cache. + hiveConf.setVar(HiveConf.ConfVars.LLAP_IO_ETL_SKIP_FORMAT, "none"); + qc.runCompactionQueries(hiveConf, null, sdMock, null, null, emptyQueries, emptyQueries, emptyQueries); + Assert.assertEquals("none", hiveConf.getVar(HiveConf.ConfVars.LLAP_IO_ETL_SKIP_FORMAT)); + + } + private class TestDataProvider { private void createTable(String tblName, boolean isPartitioned, boolean isBucketed) throws Exception { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java index 9896df3bb63bbba3685b82837fb8a9ce410f8d9c..3cdad8554c25e8d13dd5f2a4507f5d45f6b4e8a4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java @@ -96,6 +96,7 @@ protected abstract void commitCompaction(String dest, String tmpTableName, HiveC protected void runCompactionQueries(HiveConf conf, String tmpTableName, StorageDescriptor storageDescriptor, ValidWriteIdList writeIds, CompactionInfo compactionInfo, List createQueries, List compactionQueries, List dropQueries) throws IOException { + Util.disableLlapCaching(conf); String user = UserGroupInformation.getCurrentUser().getShortUserName(); SessionState sessionState = DriverUtils.setUpSessionState(conf, user, true); long compactorTxnId = CompactorMR.CompactorMap.getCompactorTxnId(conf); @@ -276,5 +277,19 @@ static void moveContents(Path sourcePath, Path destPath, boolean isMajorCompacti fs.delete(sourcePath, true); } + /** + * Unless caching is explicitly required for ETL queries this method disables it. + * LLAP cache content lookup is file based, and since compaction alters the file structure it is not beneficial to + * cache anything here, as it won't (and actually can't) ever be looked up later. + * @param conf the Hive configuration + */ + static void disableLlapCaching(HiveConf conf) { + String llapIOETLSkipFormat = conf.getVar(HiveConf.ConfVars.LLAP_IO_ETL_SKIP_FORMAT); + if (!"none".equals(llapIOETLSkipFormat)) { + // Unless caching is explicitly required for ETL queries - disable it. + conf.setVar(HiveConf.ConfVars.LLAP_IO_ETL_SKIP_FORMAT, "all"); + } + } + } }