From aeaffc0aa4d93d57dc5dd4c922d983b847cad9f2 Mon Sep 17 00:00:00 2001 From: salyh Date: Fri, 11 Jul 2014 20:51:42 +0200 Subject: [PATCH 02/14] fixed unittests for non-utf8 envs and added some more tests Signed-off-by: salyh --- .../org/apache/fleece/core/JsonReaderImplTest.java | 276 ++++++++++++++++----- 1 file changed, 217 insertions(+), 59 deletions(-) diff --git a/fleece-core/src/test/java/org/apache/fleece/core/JsonReaderImplTest.java b/fleece-core/src/test/java/org/apache/fleece/core/JsonReaderImplTest.java index 0672587..b522d26 100644 --- a/fleece-core/src/test/java/org/apache/fleece/core/JsonReaderImplTest.java +++ b/fleece-core/src/test/java/org/apache/fleece/core/JsonReaderImplTest.java @@ -18,8 +18,19 @@ */ package org.apache.fleece.core; -import org.junit.Before; -import org.junit.Test; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import javax.json.Json; import javax.json.JsonArray; @@ -27,18 +38,19 @@ import javax.json.JsonObject; import javax.json.JsonReader; import javax.json.JsonStructure; -import java.util.HashMap; - -import static org.hamcrest.CoreMatchers.instanceOf; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; +import org.junit.Test; public class JsonReaderImplTest { + + @SuppressWarnings("unchecked") + protected Map getFactoryConfig() { + return Collections.EMPTY_MAP; + } + @Test public void simple() { - final JsonReader reader = Json.createReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("json/simple.json")); + final JsonReader reader = Json.createReaderFactory(getFactoryConfig()).createReader( + Thread.currentThread().getContextClassLoader().getResourceAsStream("json/simple.json"), StandardCharsets.UTF_8); assertNotNull(reader); final JsonObject object = reader.readObject(); assertNotNull(object); @@ -53,11 +65,33 @@ public class JsonReaderImplTest { assertEquals(-2, array.getInt(1)); reader.close(); } - - + @Test public void unicode() { - final JsonReader reader = Json.createReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("json/unicode.json")); + final JsonReader reader = Json.createReaderFactory(getFactoryConfig()).createReader( + Thread.currentThread().getContextClassLoader().getResourceAsStream("json/unicode.json"), StandardCharsets.UTF_8); + assertNotNull(reader); + final JsonObject object = reader.readObject(); + assertNotNull(object); + assertEquals(String.valueOf('\u6565'), object.getString("a")); + assertEquals("", object.getString("z")); + assertEquals(String.valueOf('\u0000'), object.getString("c")); + assertThat(object.get("d"), instanceOf(JsonArray.class)); + final JsonArray array = object.getJsonArray("d"); + assertNotNull(array); + assertEquals(3, array.size()); + assertEquals(-2, array.getInt(0)); + assertEquals(" ", array.getString(1)); + assertEquals("", array.getString(2)); + assertEquals(5, object.size()); + reader.close(); + } + + @Test + public void unicodeWithIoReader() { + final Reader ioReader = new InputStreamReader(Thread.currentThread().getContextClassLoader() + .getResourceAsStream("json/unicode.json"), StandardCharsets.UTF_8); + final JsonReader reader = Json.createReader(ioReader); assertNotNull(reader); final JsonObject object = reader.readObject(); assertNotNull(object); @@ -74,15 +108,15 @@ public class JsonReaderImplTest { assertEquals(5, object.size()); reader.close(); } - + @Test public void special() { - final JsonReader reader = Json.createReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("json/special.json")); + final JsonReader reader = Json.createReaderFactory(getFactoryConfig()).createReader( + Thread.currentThread().getContextClassLoader().getResourceAsStream("json/special.json"), StandardCharsets.UTF_8); assertNotNull(reader); final JsonObject object = reader.readObject(); assertNotNull(object); assertEquals(9, object.size()); - assertEquals("b,,", object.getString("a{")); assertEquals(":4::,[{", object.getString("c::::")); assertTrue(object.getJsonNumber("w").doubleValue() > 4 && object.getJsonNumber("w").doubleValue() < 5); @@ -91,68 +125,182 @@ public class JsonReaderImplTest { assertEquals("ন:4::,[{", object.getString("থii:üäöÖ.,;.-<>!§$%&()=?ß´'`*+#")); reader.close(); } - - + + @Test + public void specialWithIoReader() { + final Reader ioReader = new InputStreamReader(Thread.currentThread().getContextClassLoader() + .getResourceAsStream("json/special.json"), StandardCharsets.UTF_8); + final JsonReader reader = Json.createReader(ioReader); + assertNotNull(reader); + final JsonObject object = reader.readObject(); + assertNotNull(object); + assertEquals(9, object.size()); + assertEquals("b,,", object.getString("a{")); + assertEquals(":4::,[{", object.getString("c::::")); + assertTrue(object.getJsonNumber("w").doubleValue() > 4 && object.getJsonNumber("w").doubleValue() < 5); + assertEquals(110, object.getInt("1.4312")); + assertEquals("\"", object.getString("\"")); + assertEquals("ন:4::,[{", object.getString("থii:üäöÖ.,;.-<>!§$%&()=?ß´'`*+#")); + reader.close(); + } + + @Test + public void specialWithStringAsByteArrayInputStream() { + final String s = "{\"নa\":\"hallo\u20acö\uffff \u08a5 থ?ߧ$%&´'`*+#\udbff\udfff\"}"; + final JsonReader reader = Json.createReaderFactory(getFactoryConfig()).createReader( + new ByteArrayInputStream(s.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8); + assertNotNull(reader); + final JsonObject object = reader.readObject(); + assertNotNull(object); + assertEquals(1, object.size()); + assertEquals("hallo\u20acö\uffff \u08a5 থ?ߧ$%&´'`*+#\udbff\udfff", object.getString("নa")); + reader.close(); + } + + @Test + public void specialWithStringReader() { + final String s = "{\"ন:4::,[{\u08a5\":\"থii:üäöÖ.,;.-<>!§$%&()=?ß´'`*+#\ua5a5\"}"; + final JsonReader reader = Json.createReaderFactory(getFactoryConfig()).createReader( + new InputStreamReader(new ByteArrayInputStream(s.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8)); + assertNotNull(reader); + final JsonObject object = reader.readObject(); + assertNotNull(object); + assertEquals(1, object.size()); + assertEquals("থii:üäöÖ.,;.-<>!§$%&()=?ß´'`*+#\ua5a5", object.getString("ন:4::,[{\u08a5")); + reader.close(); + } + + @Test + public void unicode4Bytes() { + final int codepoint = 128149; + final char[] charPair = Character.toChars(codepoint); + assertNotNull(charPair); + assertEquals(2, charPair.length); + assertTrue(Character.isHighSurrogate(charPair[0])); + assertTrue(Character.isLowSurrogate(charPair[1])); + assertTrue(Character.isSurrogatePair(charPair[0], charPair[1])); + final JsonReader reader = Json.createReaderFactory(getFactoryConfig()).createReader( + (new ByteArrayInputStream(("{\"\":\"Ö" + charPair[0] + charPair[1] + "\"}").getBytes(StandardCharsets.UTF_8))), + StandardCharsets.UTF_8); + assertNotNull(reader); + final JsonObject object = reader.readObject(); + assertNotNull(object); + + assertEquals(codepoint, object.getString("").codePointAt(1)); + assertEquals("Ö" + new String(charPair), object.getString("")); + assertEquals(1, object.size()); + reader.close(); + } + + @Test + public void unicode3Bytes() { + final char[] charPair = Character.toChars("\uffff".codePointAt(0)); + assertNotNull(charPair); + assertEquals(1, charPair.length); + assertTrue(!Character.isSurrogate(charPair[0])); + final JsonReader reader = Json.createReaderFactory(getFactoryConfig()).createReader( + new ByteArrayInputStream(("{\"\":\"\uffff\"}").getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8); + assertNotNull(reader); + final JsonObject object = reader.readObject(); + assertNotNull(object); + assertEquals(String.valueOf('\uffff'), object.getString("")); + assertEquals(1, object.size()); + reader.close(); + } + + @Test + public void unicode2Bytes() { + final JsonReader reader = Json.createReaderFactory(getFactoryConfig()).createReader( + new ByteArrayInputStream(("{\"\":\"Ö\u00d6\"}").getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8); + assertNotNull(reader); + final JsonObject object = reader.readObject(); + assertNotNull(object); + assertEquals("Ö\u00d6", object.getString("")); + assertEquals(1, object.size()); + reader.close(); + } + + @Test(expected = NullPointerException.class) + public void unicodeFailAscii() { + final JsonReader reader = Json.createReaderFactory(getFactoryConfig()).createReader( + new ByteArrayInputStream( + "{\"ন:4::,[{\udbff\udfff\":\"থii:üäöÖ.,;.-<>!§$%&()=?ß´'`*+#\udbff\udfff\"}".getBytes(StandardCharsets.US_ASCII)), + StandardCharsets.UTF_8); + assertNotNull(reader); + final JsonObject object = reader.readObject(); + assertNotNull(object); + assertEquals(1, object.size()); + assertEquals("থii:üäöÖ.,;.-<>!§$%&()=?ß´'`*+#\udbff\udfff", object.getString("ন:4::,[{\udbff\udfff")); + reader.close(); + } + @Test public void parseHuge1MbJsonFile() { - final JsonReader reader = Json.createReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("bench/huge_1mb.json")); + final JsonReader reader = Json.createReaderFactory(getFactoryConfig()).createReader( + Thread.currentThread().getContextClassLoader().getResourceAsStream("bench/huge_1mb.json"), StandardCharsets.UTF_8); assertNotNull(reader); final JsonStructure object = reader.read(); assertNotNull(object); reader.close(); } - + @Test public void parseBig600KbJsonFile() { - final JsonReader reader = Json.createReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("bench/big_600kb.json")); + final JsonReader reader = Json.createReaderFactory(getFactoryConfig()).createReader( + Thread.currentThread().getContextClassLoader().getResourceAsStream("bench/big_600kb.json"), StandardCharsets.UTF_8); assertNotNull(reader); final JsonStructure object = reader.read(); assertNotNull(object); reader.close(); } - + @Test public void parseLarge130KbJsonFile() { - final JsonReader reader = Json.createReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("bench/large_130kb.json")); + final JsonReader reader = Json.createReaderFactory(getFactoryConfig()).createReader( + Thread.currentThread().getContextClassLoader().getResourceAsStream("bench/large_130kb.json"), StandardCharsets.UTF_8); assertNotNull(reader); final JsonStructure object = reader.read(); assertNotNull(object); reader.close(); } - + @Test public void parseMedium11KbJsonFile() { - final JsonReader reader = Json.createReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("bench/medium_11kb.json")); + final JsonReader reader = Json.createReaderFactory(getFactoryConfig()).createReader( + Thread.currentThread().getContextClassLoader().getResourceAsStream("bench/medium_11kb.json"), StandardCharsets.UTF_8); assertNotNull(reader); final JsonStructure object = reader.read(); assertNotNull(object); reader.close(); } - + @Test public void parseSmall3KbJsonFile() { - final JsonReader reader = Json.createReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("bench/small_3kb.json")); + final JsonReader reader = Json.createReaderFactory(getFactoryConfig()).createReader( + Thread.currentThread().getContextClassLoader().getResourceAsStream("bench/small_3kb.json"), StandardCharsets.UTF_8); assertNotNull(reader); final JsonStructure object = reader.read(); assertNotNull(object); reader.close(); } - + @Test public void parseTiny50BJsonFile() { - final JsonReader reader = Json.createReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("bench/tiny_50b.json")); + final JsonReader reader = Json.createReaderFactory(getFactoryConfig()).createReader( + Thread.currentThread().getContextClassLoader().getResourceAsStream("bench/tiny_50b.json"), StandardCharsets.UTF_8); assertNotNull(reader); final JsonStructure object = reader.read(); assertNotNull(object); reader.close(); } - - + @Test public void simpleBadBufferSize8() { - final JsonReader reader = Json.createReaderFactory(new HashMap() {{ - put("org.apache.fleece.default-char-buffer", "8"); - }}).createReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("json/simple.json")); + final JsonReader reader = Json.createReaderFactory(new HashMap() { + { + put("org.apache.fleece.default-char-buffer", "8"); + } + }).createReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("json/simple.json"), StandardCharsets.UTF_8); assertNotNull(reader); final JsonObject object = reader.readObject(); assertNotNull(object); @@ -167,11 +315,14 @@ public class JsonReaderImplTest { assertEquals(-2, array.getInt(1)); reader.close(); } + @Test public void simpleBadBufferSize9() { - final JsonReader reader = Json.createReaderFactory(new HashMap() {{ - put("org.apache.fleece.default-char-buffer", "9"); - }}).createReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("json/simple.json")); + final JsonReader reader = Json.createReaderFactory(new HashMap() { + { + put("org.apache.fleece.default-char-buffer", "9"); + } + }).createReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("json/simple.json"), StandardCharsets.UTF_8); assertNotNull(reader); final JsonObject object = reader.readObject(); assertNotNull(object); @@ -186,53 +337,60 @@ public class JsonReaderImplTest { assertEquals(-2, array.getInt(1)); reader.close(); } - - @Test(expected=IllegalArgumentException.class) + + @Test(expected = IllegalArgumentException.class) public void emptyZeroCharBuffersize() { - final JsonReader reader = Json.createReaderFactory(new HashMap() {{ - put("org.apache.fleece.default-char-buffer", "0"); - }}).createReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("json/empty.json")); + final JsonReader reader = Json.createReaderFactory(new HashMap() { + { + put("org.apache.fleece.default-char-buffer", "0"); + } + }).createReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("json/empty.json"), StandardCharsets.UTF_8); assertNotNull(reader); reader.readObject(); reader.close(); } - + @Test public void emptyOneCharBufferSize() { - final JsonReader reader = Json.createReaderFactory(new HashMap() {{ - put("org.apache.fleece.default-char-buffer", "1"); - }}).createReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("json/empty.json")); + final JsonReader reader = Json.createReaderFactory(new HashMap() { + { + put("org.apache.fleece.default-char-buffer", "1"); + } + }).createReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("json/empty.json"), StandardCharsets.UTF_8); assertNotNull(reader); final JsonObject object = reader.readObject(); assertNotNull(object); assertEquals(0, object.size()); reader.close(); } - + @Test public void emptyArrayOneCharBufferSize() { - final JsonReader reader = Json.createReaderFactory(new HashMap() {{ - put("org.apache.fleece.default-char-buffer", "1"); - }}).createReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("json/emptyarray.json")); + final JsonReader reader = Json.createReaderFactory(new HashMap() { + { + put("org.apache.fleece.default-char-buffer", "1"); + } + }).createReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("json/emptyarray.json"), StandardCharsets.UTF_8); assertNotNull(reader); final JsonArray array = reader.readArray(); assertNotNull(array); assertEquals(0, array.size()); reader.close(); } - - + @Test public void stringescapeVariousBufferSizes() { - - int[] buffersizes = new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, - 28, 32, 64, 128, 1024, 8192 }; + final int[] buffersizes = new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, + 26, 27, 28, 32, 64, 128, 1024, 8192 }; for (int i = 0; i < buffersizes.length; i++) { final String value = String.valueOf(buffersizes[i]); - final JsonReader reader = Json.createReaderFactory(new HashMap() {{ - put("org.apache.fleece.default-char-buffer", value); - }}).createReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("json/stringescape.json")); + final JsonReader reader = Json.createReaderFactory(new HashMap() { + { + put("org.apache.fleece.default-char-buffer", value); + } + }).createReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("json/stringescape.json"), + StandardCharsets.UTF_8); assertNotNull(reader); final JsonObject object = reader.readObject(); assertNotNull(object); @@ -241,4 +399,4 @@ public class JsonReaderImplTest { reader.close(); } } -} +} \ No newline at end of file -- 1.8.5.2 (Apple Git-48)