Uploaded image for project: 'Apache Hudi'
  1. Apache Hudi
  2. HUDI-1658 [UMBRELLA] Spark Sql Support For Hudi
  3. HUDI-2232

[SQL] MERGE INTO fails with table having nested struct

Attach filesAttach ScreenshotVotersWatch issueWatchersLinkCloneUpdate Comment AuthorReplace String in CommentUpdate Comment VisibilityDelete Comments
    XMLWordPrintableJSON

    Details

      Description

      // TO reproduce
      drop table if exists hudi_gh_ext_fixed;
      
      create table hudi_gh_ext_fixed (  id int,   name string,   price double,   ts long,   repo struct<id:bigint,name:string>) using hudi options(primaryKey = 'id', precombineField = 'ts') location 'file:///tmp/hudi-h5-fixed';
      
      insert into hudi_gh_ext_fixed values(3, 'AMZN', 300, 120, struct(234273476,"onnet/onnet-portal"));
      insert into hudi_gh_ext_fixed values(2, 'UBER', 300, 120, struct(234273476,"onnet/onnet-portal"));
      insert into hudi_gh_ext_fixed values(4, 'GOOG', 300, 120, struct(234273476,"onnet/onnet-portal"));
      
      update hudi_gh_ext_fixed set price = 150.0 where name = 'UBER';
      
      drop table if exists hudi_fixed;
      
      create table hudi_fixed (  id int,   name string,   price double,   ts long,   repo struct<id:bigint,name:string>) using hudi options(primaryKey = 'id', precombineField = 'ts') partitioned by (ts) location 'file:///tmp/hudi-h5-part-fixed';
      
      insert into hudi_fixed values(2, 'UBER', 200, struct(234273476,"onnet/onnet-portal"), 130);
      
      select * from hudi_gh_ext_fixed;
      
      20210727145240  20210727145240_0_6442266  id:3    77fc2e3e-add9-4f08-a5e1-9671d66add26-0_0-1472-72063_20210727145240.parquet  3 AMZN  300.0 120 {"id":234273476,"name":"onnet/onnet-portal"}20210727145301  20210727145301_0_6442269  id:2    77fc2e3e-add9-4f08-a5e1-9671d66add26-0_0-1565-77094_20210727145301.parquet  2 UBER  150.0 120 {"id":234273476,"name":"onnet/onnet-portal"}20210727145254  20210727145254_0_6442268  id:4    77fc2e3e-add9-4f08-a5e1-9671d66add26-0_0-1534-75283_20210727145254.parquet  4 GOOG  300.0 120 {"id":234273476,"name":"onnet/onnet-portal"}
      
      select * from hudi_fixed;
      
      20210727145325  20210727145325_0_6442270  id:2  ts=130  ba148271-68b4-40aa-816a-158170446e41-0_0-1595-78703_20210727145325.parquet  2 UBER  200.0 {"id":234273476,"name":"onnet/onnet-portal"}  130
      
      MERGE INTO hudi_fixed USING (select id, name, price, repo, ts from hudi_gh_ext_fixed) updatesON hudi_fixed.id = updates.idWHEN MATCHED THEN  UPDATE SET *WHEN NOT MATCHED  THEN INSERT *;
      
      -- java.lang.IllegalArgumentException: UnSupport StructType yet--  at org.apache.spark.sql.hudi.command.payload.SqlTypedRecord.convert(SqlTypedRecord.scala:122)--  at org.apache.spark.sql.hudi.command.payload.SqlTypedRecord.get(SqlTypedRecord.scala:56)--  at org.apache.hudi.sql.payload.ExpressionPayloadEvaluator_b695b02a_99b5_479e_8299_507da9b206fd.eval(Unknown Source)--  at org.apache.spark.sql.hudi.command.payload.ExpressionPayload$AvroTypeConvertEvaluator.eval(ExpressionPayload.scala:333)
      

        Attachments

          Activity

            People

            • Assignee:
              pzw2018 pengzhiwei
              Reporter:
              codope Sagar Sumit

              Dates

              • Created:
                Updated:
                Resolved:

                Issue deployment