diff --git itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java index 3c0532c9d0..460f1d03a7 100644 --- itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java +++ itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java @@ -250,6 +250,53 @@ public void testLlapInputFormatEndToEndWithMultipleBatches() throws Exception { assertEquals(0, rowCount); } + @Test(timeout = 300000) + public void testInvalidReferenceCountScenario() throws Exception { + final String tableName = "testInvalidReferenceCountScenario"; + try (Statement stmt = hs2Conn.createStatement()) { + String createQuery = + "create table " + tableName + + "(arr1 array>>>, " + + "c2 int) STORED AS ORC"; + + // create table + stmt.execute("DROP TABLE IF EXISTS " + tableName); + stmt.execute(createQuery); + // load data + stmt.execute("INSERT INTO " + tableName + " VALUES " + // value 1 + + "(ARRAY(NAMED_STRUCT('f1','a1', " + + "'f2','a2', " + + "'arr2'," + + " ARRAY(" + + "NAMED_STRUCT('f3', cast(null as string), 'f4', cast(null as string), 'f5', cast(null as string)))), " + + "NAMED_STRUCT('f1','a1', 'f2','a2', 'arr2', " + + "ARRAY(NAMED_STRUCT('f3', 'fielddddddd3333333', 'f4', 'field4', 'f5', 'field5'))), " + + "NAMED_STRUCT('f1','a1', 'f2','a2', 'arr2', ARRAY(NAMED_STRUCT('f3', cast(null as string), " + + "'f4', cast(null as string), 'f5', cast(null as string)))), " + + "NAMED_STRUCT('f1','a1', 'f2','a2', 'arr2', ARRAY(NAMED_STRUCT('f3', 'fielddddddd3333333', " + + "'f4', 'field4', 'f5', 'field5'))), NAMED_STRUCT('f1','a1', 'f2','a2', 'arr2', " + + "ARRAY(NAMED_STRUCT('f3', cast(null as string), 'f4', cast(null as string), 'f5', cast(null as string))))," + + " NAMED_STRUCT('f1','a1', 'f2','a2', 'arr2', " + + "ARRAY(NAMED_STRUCT('f3', 'fielddddddd3333333', 'f4', 'field4', 'f5', 'field5')))), 1)"); + + // generate 16384 rows from above records + for (int i = 0; i < 14; i++) { + stmt.execute(String.format("insert into %s select * from %s", tableName, tableName)); + } + // validate test table + ResultSet res = stmt.executeQuery("SELECT count(*) FROM " + tableName); + assertTrue(res.next()); + assertEquals(16384, res.getInt(1)); + res.close(); + } + // should not throw - IllegalReferenceCountException: refCnt: 0 + RowCollector rowCollector = new RowCollector(); + String query = "select * from " + tableName; + int rowCount = processQuery(query, 1, rowCollector); + assertEquals(16384, rowCount); + + } @Test(timeout = 60000) public void testNonAsciiStrings() throws Exception { diff --git ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java index 035097759a..12da074e76 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java @@ -313,16 +313,15 @@ private void writeMap(ListVector arrowVector, MapColumnVector hiveVector, MapTyp write(arrowVector, structListVector, structListTypeInfo, size, vectorizedRowBatch, isNative); - final ArrowBuf validityBuffer = arrowVector.getValidityBuffer(); for (int rowIndex = 0; rowIndex < size; rowIndex++) { int selectedIndex = rowIndex; if (vectorizedRowBatch.selectedInUse) { selectedIndex = vectorizedRowBatch.selected[rowIndex]; } if (hiveVector.isNull[selectedIndex]) { - BitVectorHelper.setValidityBit(validityBuffer, rowIndex, 0); + BitVectorHelper.setValidityBit(arrowVector.getValidityBuffer(), rowIndex, 0); } else { - BitVectorHelper.setValidityBitToOne(validityBuffer, rowIndex); + BitVectorHelper.setValidityBitToOne(arrowVector.getValidityBuffer(), rowIndex); } } } @@ -360,12 +359,11 @@ private void writeStruct(NonNullableStructVector arrowVector, StructColumnVector write(arrowFieldVector, hiveFieldVector, fieldTypeInfo, size, vectorizedRowBatch, isNative); } - final ArrowBuf validityBuffer = arrowVector.getValidityBuffer(); for (int rowIndex = 0; rowIndex < size; rowIndex++) { if (hiveVector.isNull[rowIndex]) { - BitVectorHelper.setValidityBit(validityBuffer, rowIndex, 0); + BitVectorHelper.setValidityBit(arrowVector.getValidityBuffer(), rowIndex, 0); } else { - BitVectorHelper.setValidityBitToOne(validityBuffer, rowIndex); + BitVectorHelper.setValidityBitToOne(arrowVector.getValidityBuffer(), rowIndex); } } } @@ -426,7 +424,6 @@ private void writeList(ListVector arrowVector, ListColumnVector hiveVector, List write(arrowElementVector, hiveElementVector, elementTypeInfo, correctedSize, correctedVrb, isNative); - final ArrowBuf offsetBuffer = arrowVector.getOffsetBuffer(); int nextOffset = 0; for (int rowIndex = 0; rowIndex < size; rowIndex++) { @@ -435,14 +432,14 @@ private void writeList(ListVector arrowVector, ListColumnVector hiveVector, List selectedIndex = vectorizedRowBatch.selected[rowIndex]; } if (hiveVector.isNull[selectedIndex]) { - offsetBuffer.setInt(rowIndex * OFFSET_WIDTH, nextOffset); + arrowVector.getOffsetBuffer().setInt(rowIndex * OFFSET_WIDTH, nextOffset); } else { - offsetBuffer.setInt(rowIndex * OFFSET_WIDTH, nextOffset); + arrowVector.getOffsetBuffer().setInt(rowIndex * OFFSET_WIDTH, nextOffset); nextOffset += (int) hiveVector.lengths[selectedIndex]; arrowVector.setNotNull(rowIndex); } } - offsetBuffer.setInt(size * OFFSET_WIDTH, nextOffset); + arrowVector.getOffsetBuffer().setInt(size * OFFSET_WIDTH, nextOffset); } //Handle cases for both internally constructed