From c37acb7bdab47e2c5d2dc73a04c0e13e6b2f8902 Mon Sep 17 00:00:00 2001 From: Tmin10 Date: Thu, 23 Aug 2018 16:07:19 +0300 Subject: [PATCH] HIVE-20447 Add JSON Outputformat support --- .../hive/beeline/AbstractOutputFormat.java | 5 + .../java/org/apache/hive/beeline/BeeLine.java | 1 + .../apache/hive/beeline/JsonOutputFormat.java | 163 ++++++++++++++++++ .../beeline/XMLAttributeOutputFormat.java | 3 +- .../hive/beeline/XMLElementOutputFormat.java | 3 +- beeline/src/main/resources/BeeLine.properties | 2 +- .../hive/beeline/TestJsonOutputFormat.java | 134 ++++++++++++++ 7 files changed, 306 insertions(+), 5 deletions(-) create mode 100644 beeline/src/java/org/apache/hive/beeline/JsonOutputFormat.java create mode 100644 beeline/src/test/org/apache/hive/beeline/TestJsonOutputFormat.java diff --git a/beeline/src/java/org/apache/hive/beeline/AbstractOutputFormat.java b/beeline/src/java/org/apache/hive/beeline/AbstractOutputFormat.java index b0e2357d2f..efdb31d259 100644 --- a/beeline/src/java/org/apache/hive/beeline/AbstractOutputFormat.java +++ b/beeline/src/java/org/apache/hive/beeline/AbstractOutputFormat.java @@ -27,6 +27,11 @@ * */ abstract class AbstractOutputFormat implements OutputFormat { + protected final BeeLine beeLine; + + public AbstractOutputFormat(BeeLine beeLine) { + this.beeLine = beeLine; + } public int print(Rows rows) { int count = 0; diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java index 4eda8e3ff5..cd08bbd280 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java @@ -184,6 +184,7 @@ "tsv", new DeprecatedSeparatedValuesOutputFormat(this, '\t'), "xmlattr", new XMLAttributeOutputFormat(this), "xmlelements", new XMLElementOutputFormat(this), + "json", new JsonOutputFormat(this) }); private List supportedLocalDriver = diff --git a/beeline/src/java/org/apache/hive/beeline/JsonOutputFormat.java b/beeline/src/java/org/apache/hive/beeline/JsonOutputFormat.java new file mode 100644 index 0000000000..b3d096b0b8 --- /dev/null +++ b/beeline/src/java/org/apache/hive/beeline/JsonOutputFormat.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This source file is based on code taken from SQLLine 1.4.0 + * See SQLLine notice in LICENSE + */ +package org.apache.hive.beeline; + +import java.sql.SQLException; +import java.sql.Types; +import java.util.HashMap; +import java.util.Map; + +/** + * Implementation of {@link OutputFormat} that formats rows as JSON. + */ +public class JsonOutputFormat extends AbstractOutputFormat { + private static final Map ESCAPING_MAP = new HashMap(); + + static { + ESCAPING_MAP.put('\\', "\\\\"); + ESCAPING_MAP.put('\"', "\\\""); + ESCAPING_MAP.put('\b', "\\b"); + ESCAPING_MAP.put('\f', "\\f"); + ESCAPING_MAP.put('\n', "\\n"); + ESCAPING_MAP.put('\r', "\\r"); + ESCAPING_MAP.put('\t', "\\t"); + ESCAPING_MAP.put('/', "\\/"); + ESCAPING_MAP.put('\u0000', "\\u0000"); + ESCAPING_MAP.put('\u0001', "\\u0001"); + ESCAPING_MAP.put('\u0002', "\\u0002"); + ESCAPING_MAP.put('\u0003', "\\u0003"); + ESCAPING_MAP.put('\u0004', "\\u0004"); + ESCAPING_MAP.put('\u0005', "\\u0005"); + ESCAPING_MAP.put('\u0006', "\\u0006"); + ESCAPING_MAP.put('\u0007', "\\u0007"); + // ESCAPING_MAP.put('\u0008', "\\u0008"); + // covered by ESCAPING_MAP.put('\b', "\\b"); + // ESCAPING_MAP.put('\u0009', "\\u0009"); + // covered by ESCAPING_MAP.put('\t', "\\t"); + // ESCAPING_MAP.put((char) 10, "\\u000A"); + // covered by ESCAPING_MAP.put('\n', "\\n"); + ESCAPING_MAP.put('\u000B', "\\u000B"); + // ESCAPING_MAP.put('\u000C', "\\u000C"); + // covered by ESCAPING_MAP.put('\f', "\\f"); + // ESCAPING_MAP.put((char) 13, "\\u000D"); + // covered by ESCAPING_MAP.put('\r', "\\r"); + ESCAPING_MAP.put('\u000E', "\\u000E"); + ESCAPING_MAP.put('\u000F', "\\u000F"); + ESCAPING_MAP.put('\u0010', "\\u0010"); + ESCAPING_MAP.put('\u0011', "\\u0011"); + ESCAPING_MAP.put('\u0012', "\\u0012"); + ESCAPING_MAP.put('\u0013', "\\u0013"); + ESCAPING_MAP.put('\u0014', "\\u0014"); + ESCAPING_MAP.put('\u0015', "\\u0015"); + ESCAPING_MAP.put('\u0016', "\\u0016"); + ESCAPING_MAP.put('\u0017', "\\u0017"); + ESCAPING_MAP.put('\u0018', "\\u0018"); + ESCAPING_MAP.put('\u0019', "\\u0019"); + ESCAPING_MAP.put('\u001A', "\\u001A"); + ESCAPING_MAP.put('\u001B', "\\u001B"); + ESCAPING_MAP.put('\u001C', "\\u001C"); + ESCAPING_MAP.put('\u001D', "\\u001D"); + ESCAPING_MAP.put('\u001E', "\\u001E"); + ESCAPING_MAP.put('\u001F', "\\u001F"); + } + + private int[] columnTypes; + + public JsonOutputFormat(BeeLine beeLine) { + super(beeLine); + } + + @Override + void printHeader(Rows.Row header) { + beeLine.output("{\"resultset\":["); + } + + @Override + void printFooter(Rows.Row header) { + beeLine.output("]}"); + } + + @Override + void printRow(Rows rows, Rows.Row header, Rows.Row row) { + String[] head = header.values; + String[] vals = row.values; + StringBuilder sb = new StringBuilder("{"); + for (int i = 0; (i < head.length) && (i < vals.length); i++) { + if (columnTypes == null) { + initColumnTypes(rows, header); + } + sb.append("\"").append(head[i]).append("\":"); + setJsonValue(sb, vals[i], columnTypes[i]); + sb.append((i < head.length - 1) && (i < vals.length - 1) ? "," : ""); + } + sb.append(rows.hasNext() ? "}," : "}"); + beeLine.output(sb.toString()); + } + + private void setJsonValue(StringBuilder sb, String value, int columnTypeId) { + if (value == null) { + sb.append(value); + return; + } + switch (columnTypeId) { + case Types.TINYINT: + case Types.SMALLINT: + case Types.INTEGER: + case Types.BIGINT: + case Types.REAL: + case Types.FLOAT: + case Types.DOUBLE: + case Types.DECIMAL: + case Types.NUMERIC: + sb.append(value); + return; + case Types.NULL: + sb.append(value.toLowerCase()); + return; + case Types.BOOLEAN: + // JSON requires true and false, not TRUE and FALSE + sb.append(value.equalsIgnoreCase("TRUE")); + return; + } + sb.append("\""); + for (int i = 0; i < value.length(); i++) { + if (ESCAPING_MAP.get(value.charAt(i)) != null) { + sb.append(ESCAPING_MAP.get(value.charAt(i))); + } else { + sb.append(value.charAt(i)); + } + } + sb.append("\""); + } + + private void initColumnTypes(Rows rows, Rows.Row header) { + columnTypes = new int[header.values.length]; + for (int j = 0; j < header.values.length; j++) { + try { + columnTypes[j] = rows.rsMeta.getColumnType(j + 1); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + } +} \ No newline at end of file diff --git a/beeline/src/java/org/apache/hive/beeline/XMLAttributeOutputFormat.java b/beeline/src/java/org/apache/hive/beeline/XMLAttributeOutputFormat.java index 87755f0977..5708363d71 100644 --- a/beeline/src/java/org/apache/hive/beeline/XMLAttributeOutputFormat.java +++ b/beeline/src/java/org/apache/hive/beeline/XMLAttributeOutputFormat.java @@ -23,14 +23,13 @@ package org.apache.hive.beeline; class XMLAttributeOutputFormat extends AbstractOutputFormat { - private final BeeLine beeLine; private final StringBuilder buf = new StringBuilder(); /** * @param beeLine */ XMLAttributeOutputFormat(BeeLine beeLine) { - this.beeLine = beeLine; + super(beeLine); } @Override diff --git a/beeline/src/java/org/apache/hive/beeline/XMLElementOutputFormat.java b/beeline/src/java/org/apache/hive/beeline/XMLElementOutputFormat.java index 68ef9ef5b7..b3d2ee1b55 100644 --- a/beeline/src/java/org/apache/hive/beeline/XMLElementOutputFormat.java +++ b/beeline/src/java/org/apache/hive/beeline/XMLElementOutputFormat.java @@ -23,13 +23,12 @@ package org.apache.hive.beeline; class XMLElementOutputFormat extends AbstractOutputFormat { - private final BeeLine beeLine; /** * @param beeLine */ XMLElementOutputFormat(BeeLine beeLine) { - this.beeLine = beeLine; + super(beeLine); } @Override diff --git a/beeline/src/main/resources/BeeLine.properties b/beeline/src/main/resources/BeeLine.properties index 30b4ef954a..da61887b54 100644 --- a/beeline/src/main/resources/BeeLine.properties +++ b/beeline/src/main/resources/BeeLine.properties @@ -68,7 +68,7 @@ help-procedures: List all the procedures help-tables: List all the tables in the database help-columns: List all the columns for the specified table help-properties: Connect to the database specified in the properties file(s) -help-outputformat: Set the output format for displaying results (table,vertical,csv2,dsv,tsv2,xmlattrs,xmlelements, and deprecated formats(csv, tsv)) +help-outputformat: Set the output format for displaying results (table,vertical,csv2,dsv,tsv2,xmlattrs,xmlelements,json and deprecated formats(csv, tsv)) help-delimiterForDSV: Set the delimiter for dsv output format help-nullemptystring: Set to true to get historic behavior of printing null as empty string. Default is false. help-addlocaldriverjar: Add driver jar file in the beeline client side. diff --git a/beeline/src/test/org/apache/hive/beeline/TestJsonOutputFormat.java b/beeline/src/test/org/apache/hive/beeline/TestJsonOutputFormat.java new file mode 100644 index 0000000000..d65ce2472f --- /dev/null +++ b/beeline/src/test/org/apache/hive/beeline/TestJsonOutputFormat.java @@ -0,0 +1,134 @@ +/* + * Copyright 2018 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.Matchers; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import java.io.PrintStream; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Types; +import java.util.ArrayList; + +import static org.junit.Assert.assertArrayEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TestJsonOutputFormat { + + private final Object[][] mockRowData = { + {"aaa", new Boolean(true), null, new Double(3.14), "\\/\b\f\n\r\t"} + }; + private TestJsonOutputFormat.BeelineMock mockBeeline; + private ResultSet mockResultSet; + private MockRow mockRow; + + @Before + public void setupMockData() throws SQLException { + mockBeeline = new TestJsonOutputFormat.BeelineMock(); + mockResultSet = mock(ResultSet.class); + + ResultSetMetaData mockResultSetMetaData = mock(ResultSetMetaData.class); + when(mockResultSetMetaData.getColumnCount()).thenReturn(5); + when(mockResultSetMetaData.getColumnLabel(1)).thenReturn("string"); + when(mockResultSetMetaData.getColumnLabel(2)).thenReturn("boolean"); + when(mockResultSetMetaData.getColumnLabel(3)).thenReturn("null"); + when(mockResultSetMetaData.getColumnLabel(4)).thenReturn("double"); + when(mockResultSetMetaData.getColumnLabel(5)).thenReturn("special symbols"); + when(mockResultSetMetaData.getColumnType(1)).thenReturn(Types.VARCHAR); + when(mockResultSetMetaData.getColumnType(2)).thenReturn(Types.BOOLEAN); + when(mockResultSetMetaData.getColumnType(3)).thenReturn(Types.NULL); + when(mockResultSetMetaData.getColumnType(4)).thenReturn(Types.DOUBLE); + when(mockResultSetMetaData.getColumnType(5)).thenReturn(Types.VARCHAR); + when(mockResultSet.getMetaData()).thenReturn(mockResultSetMetaData); + + mockRow = new MockRow(); + // returns true as long as there is more data in mockResultData array + when(mockResultSet.next()).thenAnswer(new Answer() { + private int mockRowDataIndex = 0; + + @Override + public Boolean answer(final InvocationOnMock invocation) { + if (mockRowDataIndex < mockRowData.length) { + mockRow.setCurrentRowData(mockRowData[mockRowDataIndex]); + mockRowDataIndex++; + return true; + } else { + return false; + } + } + }); + + when(mockResultSet.getObject(Matchers.anyInt())).thenAnswer(new Answer() { + @Override + public Object answer(final InvocationOnMock invocation) { + Object[] args = invocation.getArguments(); + int index = ((Integer) args[0]); + return mockRow.getColumn(index); + } + }); + } + + /** + * Test printing output data with JsonOutputFormat + */ + @Test + public final void testPrint() throws SQLException { + setupMockData(); + BufferedRows bfRows = new BufferedRows(mockBeeline, mockResultSet); + JsonOutputFormat instance = new JsonOutputFormat(mockBeeline); + instance.print(bfRows); + ArrayList actualOutput = mockBeeline.getLines(); + ArrayList expectedOutput = new ArrayList<>(6); + expectedOutput.add("{\"resultset\":["); + expectedOutput.add("{\"string\":\"aaa\"," + "\"boolean\":true," + "\"null\":null," + "\"double\":3.14," + + "\"special symbols\":\"\\\\\\/\\b\\f\\n\\r\\t\"}"); + expectedOutput.add("]}"); + assertArrayEquals(expectedOutput.toArray(), actualOutput.toArray()); + } + + public class BeelineMock extends BeeLine { + + private ArrayList lines = new ArrayList<>(); + + @Override + final void output(final ColorBuffer msg, boolean newline, PrintStream out) { + lines.add(msg.getMono()); + super.output(msg, newline, out); + } + + private ArrayList getLines() { + return lines; + } + } + + static class MockRow { + Object[] rowData; + + public void setCurrentRowData(Object[] rowData) { + this.rowData = rowData; + } + + public Object getColumn(int idx) { + return rowData[idx - 1]; + } + } +} -- 2.18.0.windows.1