[DRILL-7045] UDF string_binary java.lang.IndexOutOfBoundsException:

• Type: Bug
• Status: Resolved
• Priority: Minor
• Resolution: Fixed
• Affects Version/s: 1.15.0
• Fix Version/s: 1.16.0
• Component/s: Functions - Drill
• Labels: ready-to-commit
• Assignee: jean-claude
• Reporter: jean-claude
• Reviewer: Bohdan Kazydub
• Created: 20/Feb/19 13:06
• Updated: 08/Apr/19 10:43
• Resolved: 08/Apr/19 10:43

Description

Given a large field like

 

cat input.json

{ "col0": "lajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfj" } { "col0": "lajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfj" } { "col0": "lajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfj" }

 

drop table dfs.tmp.`converted`;
create table dfs.tmp.`converted` as (select convert_to(col0, 'UTF8') as col0_bin from dfs.root.`prog/input.json`);
select col0_bin from dfs.tmp.`converted`;
select convert_from(col0_bin, 'UTF8') from dfs.tmp.`converted`;
select string_binary(col0_bin) from dfs.tmp.`converted`;

The last statement fails with

Caused by: java.lang.IndexOutOfBoundsException: index: 0, length: 9068 (expected: range(0, 256))
at io.netty.buffer.AbstractByteBuf.checkIndex0(AbstractByteBuf.java:1125)
at io.netty.buffer.AbstractByteBuf.checkIndex(AbstractByteBuf.java:1120)
at io.netty.buffer.UnsafeByteBufUtil.setBytes(UnsafeByteBufUtil.java:349)
at io.netty.buffer.PooledUnsafeDirectByteBuf.setBytes(PooledUnsafeDirectByteBuf.java:199)
at io.netty.buffer.WrappedByteBuf.setBytes(WrappedByteBuf.java:397)
at io.netty.buffer.UnsafeDirectLittleEndian.setBytes(UnsafeDirectLittleEndian.java:36)
at io.netty.buffer.DrillBuf.setBytes(DrillBuf.java:735)
at io.netty.buffer.AbstractByteBuf.setBytes(AbstractByteBuf.java:528)
at org.apache.drill.exec.test.generated.ProjectorGen15.doEval(ProjectorTemplate.java:49)
at org.apache.drill.exec.test.generated.ProjectorGen15.projectRecords(ProjectorTemplate.java:67)
at org.apache.drill.exec.physical.impl.project.ProjectRecordBatch.doWork(ProjectRecordBatch.java:232)

 

This is probably due to the fact that the string_binary UDF does not reallocate it's buffer based on the size of the converted byte array.

public void eval() {
byte[] buf = org.apache.drill.common.util.DrillStringUtils.toBinaryString(in.buffer, in.start, in.end).getBytes(charset);
buffer.setBytes(0, buf);
buffer.setIndex(0, buf.length);

out.start = 0;
out.end = buf.length;
out.buffer = buffer;
}

is missing an

out.buffer = buffer.reallocIfNeeded(in.end - in.start);

 

 

 

Issue Links

• links to: GitHub Pull Request #1671
• links to: GitHub Pull Request #1734