commit 906f618d9b148ab839eae03940dfdfcebe6302db Author: Owen O'Malley Date: Tue Jul 12 08:31:03 2016 -0700 HIVE-14004 array out of bounds in schema evolution diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java index 9bcdb39..74c5071 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java @@ -79,6 +79,7 @@ public RecordReader rowsOptions(Options options) throws IOException { boolean[] include = options.getInclude(); // if included columns is null, then include all columns if (include == null) { + options = options.clone(); include = new boolean[types.size()]; Arrays.fill(include, true); options.include(include); diff --git ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java index e76c925..245a3bc 100644 --- ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java +++ ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java @@ -846,7 +846,7 @@ public void testInitiatorWithMultipleFailedCompactions() throws Exception { init.run(); int numAttemptedCompactions = 1; checkCompactionState(new CompactionsByState(numAttemptedCompactions,numFailedCompactions,0,0,0,0,numFailedCompactions + numAttemptedCompactions), countCompacts(txnHandler)); - + hiveConf.setTimeVar(HiveConf.ConfVars.COMPACTOR_HISTORY_REAPER_INTERVAL, 10, TimeUnit.MILLISECONDS); AcidCompactionHistoryService compactionHistoryService = new AcidCompactionHistoryService(); runHouseKeeperService(compactionHistoryService, hiveConf);//should not remove anything from history @@ -868,7 +868,7 @@ public void testInitiatorWithMultipleFailedCompactions() throws Exception { hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_ATTEMPTED), hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_FAILED),0,0,0,0, hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_FAILED) + hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_ATTEMPTED)), countCompacts(txnHandler)); - + hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEFAILCOMPACTION, false); txnHandler.compact(new CompactionRequest("default", tblName, CompactionType.MINOR)); //at this point "show compactions" should have (COMPACTOR_HISTORY_RETENTION_FAILED) failed + 1 initiated (explicitly by user) @@ -1139,6 +1139,30 @@ public void testOpenTxnsCounter() throws Exception { Assert.assertNull(exception); } + @Test + public void testCompactWithDelete() throws Exception { + int[][] tableData = {{1,2},{3,4}}; + runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData)); + runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MAJOR'"); + Worker t = new Worker(); + t.setThreadId((int) t.getId()); + t.setHiveConf(hiveConf); + AtomicBoolean stop = new AtomicBoolean(); + AtomicBoolean looped = new AtomicBoolean(); + stop.set(true); + t.init(stop, looped); + t.run(); + runStatementOnDriver("delete from " + Table.ACIDTBL + " where b = 4"); + runStatementOnDriver("update " + Table.ACIDTBL + " set b = -2 where b = 2"); + runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MINOR'"); + t.run(); + TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf); + ShowCompactResponse resp = txnHandler.showCompact(new ShowCompactRequest()); + Assert.assertEquals("Unexpected number of compactions in history", 2, resp.getCompactsSize()); + Assert.assertEquals("Unexpected 0 compaction state", TxnStore.CLEANING_RESPONSE, resp.getCompacts().get(0).getState()); + Assert.assertEquals("Unexpected 1 compaction state", TxnStore.CLEANING_RESPONSE, resp.getCompacts().get(1).getState()); + } + /** * takes raw data and turns it into a string as if from Driver.getResults() * sorts rows in dictionary order