Uploaded image for project: 'Flink'
  1. Flink
  2. FLINK-23269

json format decode number to vatchar error when define Decimal type in ddl

    XMLWordPrintableJSON

    Details

      Description

      when use json format and define decimal
      json:

      {"c1":50.0,"c2":50.0}

      ddl:
      create table(
      c1 varchar,
      c2 decimal
      )with(
      'format'='json'
      )
      output:

      {"c1":"5E+1","c2":50.0}

      And the following unit tests will produce the following results

      {"double1":50.0,"double2":50.0,"double3":"50.0","float1":20.0,"float2":20.0,"float3":"20.0"}

      java.lang.AssertionError:
      Expected :+I[50.0, 50.0, 50.0, 20.0, 20.0, 20.0]
      Actual :+I[5E+1, 50.0, 50.0, 2E+1, 20.0, 20.0]

      @Test
          public void testDeserialization() throws Exception {
              double doubleValue = 50.0;
              float floatValue = 20.0f;
      
              ObjectMapper objectMapper = new ObjectMapper();
              ObjectNode root = objectMapper.createObjectNode();
              root.put("double1", doubleValue);
              root.put("double2", doubleValue);
              root.put("double3", String.valueOf(doubleValue));
              root.put("float1", floatValue);
              root.put("float2", floatValue);
              root.put("float3", String.valueOf(floatValue));
      
              byte[] serializedJson = objectMapper.writeValueAsBytes(root);
              System.out.println(new String(serializedJson));
              DataType dataType =
                      ROW(
                              FIELD("double1", STRING()),
                              FIELD("double2", DECIMAL(10,1)),
                              FIELD("double3", DOUBLE()),
                              FIELD("float1", STRING()),
                              FIELD("float2", DECIMAL(10,1)),
                              FIELD("float3", FLOAT()));
      
              RowType rowType = (RowType) dataType.getLogicalType();
              JsonRowDataDeserializationSchema deserializationSchema =
                      new JsonRowDataDeserializationSchema(
                              rowType,
                              InternalTypeInfo.of(rowType),
                              false,
                              false,
                              TimestampFormat.ISO_8601);
      
              Row expected = new Row(6);
              expected.setField(0, String.valueOf(doubleValue));
              expected.setField(1, String.valueOf(doubleValue));
              expected.setField(2, doubleValue);
              expected.setField(3, String.valueOf(floatValue));
              expected.setField(4, String.valueOf(floatValue));
              expected.setField(5, floatValue);
      
              RowData rowData = deserializationSchema.deserialize(serializedJson);
              Row actual = convertToExternal(rowData, dataType);
              assertEquals(expected, actual);
          }
      

      when define the DecimalType
      ObjectMapper will enable USE_BIG_DECIMAL_FOR_FLOATS
      and jsonNode.asText() will call BigDecimal toString method

      boolean hasDecimalType =
                      LogicalTypeChecks.hasNested(rowType, t -> t instanceof DecimalType);
              if (hasDecimalType) {
                  objectMapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS);
              }
      

        Attachments

          Activity

            People

            • Assignee:
              Unassigned
              Reporter:
              zhouqi silence
            • Votes:
              0 Vote for this issue
              Watchers:
              3 Start watching this issue

              Dates

              • Created:
                Updated: