Uploaded image for project: 'CarbonData'
  1. CarbonData
  2. CARBONDATA-4202

Fix issue when refresh main table with MV

    XMLWordPrintableJSON

Details

    • Bug
    • Status: Resolved
    • Major
    • Resolution: Fixed
    • None
    • 2.2.0
    • None
    • None

    Description

      [Problem phenomenon] : - error when trying to refresh main table which contain mv in 2.1.1. Store for main table with mv created in 2.1.0

      [Steps] :-

      CREATE TABLE originTable_mv (empno int, empname String, designation String, doj Timestamp,workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int)STORED AS carbondata;

      LOAD DATA local inpath 'hdfs://hacluster/BabuStore/Data/data.csv' INTO TABLE originTable_mv OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"','timestampformat'='dd-MM-yyyy');

      create MATERIALIZED VIEW datamap_comp_mv as select empno,sum(attendance) ,min(projectjoindate) ,max(projectenddate) ,avg(attendance) ,count(empno),count(distinct workgroupcategoryname) from originTable_mv group by empno;

      LOAD DATA local inpath 'hdfs://hacluster/BabuStore/Data/data.csv' INTO TABLE originTable_mv OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"','timestampformat'='dd-MM-yyyy');

      Backup the store and copy the store .

      execute the refresh table command on the main table that has MV table.

      [Expected Result] :- refresh main table which contain mv in 2.1.1. Store for main table with mv created in 2.1.0 should be successful

      [Actual Issue] : -error when trying to refresh main table which contain mv in 2.1.1. Store for main table with mv created in 2.1.0

      0: jdbc:hive2://linux-221:22550/> refresh table originTable_mv;

      Error: org.apache.hive.service.cli.HiveSQLException: Error running query: org.apache.spark.sql.AnalysisException: == Spark Parser: org.apache.spark.sql.hive.FISqlParser ==

      extraneous input '2_1' expecting {')', ','}(line 8, pos 25)

      == SQL ==

      CREATE TABLE 2_1.origintable_mv

      (empno int,empname string,designation string,doj timestamp,workgroupcategory int,workgroupcategoryname string,deptno int,deptname string,projectcode int,projectjoindate timestamp,projectenddate timestamp,attendance int,utilization int,salary int)

      USING carbondata

      OPTIONS (

      indexexists "false",

      sort_columns "",

      comment "",

      relatedmvtablesmap "{"2_1":["datamap_comp_mv"]}",

      -------------------------^^^

      bad_record_path "",

      local_dictionary_enable "true",

      indextableexists "false",

      tableName "origintable_mv",

      dbName "2_1",

      tablePath "hdfs://hacluster/user/hive/warehouse/carbon.store/2_1/origintable_mv",

      path "hdfs://hacluster/user/hive/warehouse/carbon.store/2_1/origintable_mv",

      isExternal "false",

      isTransactional "true",

      isVisible "true"

      ,carbonSchemaPartsNo '2',carbonSchema0 '{"databaseName":"2_1","tableUniqueName":"2_1_origintable_mv","factTable":{"tableId":"5b9f23bf-c08a-49bf-9b33-6f1a397014e5","tableName":"origintable_mv","listOfColumns":[{"dataType":

      {"id":0,"precedenceOrder":0,"name":"STRING","sizeInBytes":-1}

      ,"columnName":"empname","columnUniqueId":"663d1b53-2898-49e5-be7c-c7ce7c70d538","columnReferenceId":"663d1b53-2898-49e5-be7c-c7ce7c70d538","encodingList":[],"isDimensionColumn":true,"scale":-1,"precision":-1,"schemaOrdinal":1,"numberOfChild":0,"invisible":false,"isSortColumn":false,"aggFunction":"","timeSeriesFunction":"","isLocalDictColumn":true},{"dataType":

      {"id":0,"precedenceOrder":0,"name":"STRING","sizeInBytes":-1}

      ,"columnName":"designation","columnUniqueId":"9f708f4b-5ce4-4169-b6db-dbe34092ded0","columnReferenceId":"9f708f4b-5ce4-4169-b6db-dbe34092ded0","encodingList":[],"isDimensionColumn":true,"scale":-1,"precision":-1,"schemaOrdinal":2,"numberOfChild":0,"invisible":false,"isSortColumn":false,"aggFunction":"","timeSeriesFunction":"","isLocalDictColumn":true},{"dataType":

      {"id":2,"precedenceOrder":2,"name":"TIMESTAMP","sizeInBytes":-1}

      ,"columnName":"doj","columnUniqueId":"2f606b57-5fc3-4f98-93b9-3332e78cb475","columnReferenceId":"2f606b57-5fc3-4f98-93b9-3332e78cb475","encodingList":[],"isDimensionColumn":true,"scale":-1,"precision":-1,"schemaOrdinal":3,"numberOfChild":0,"invisible":false,"isSortColumn":false,"aggFunction":"","timeSeriesFunction":"","isLocalDictColumn":false},{"dataType":

      {"id":0,"precedenceOrder":0,"name":"STRING","sizeInBytes":-1}

      ,"columnName":"workgroupcategoryname","columnUniqueId":"b20c49ac-e59c-4142-a7b8-6b38cc65e908","columnReferenceId":"b20c49ac-e59c-4142-a7b8-6b38cc65e908","encodingList":[],"isDimensionColumn":true,"scale":-1,"precision":-1,"schemaOrdinal":5,"numberOfChild":0,"invisible":false,"isSortColumn":false,"aggFunction":"","timeSeriesFunction":"","isLocalDictColumn":true},{"dataType":

      {"id":0,"precedenceOrder":0,"name":"STRING","sizeInBytes":-1}

      ,"columnName":"deptname","columnUniqueId":"dc6e48c9-9814-4a2b-93f9-72c726ffc8c8","columnReferenceId":"dc6e48c9-9814-4a2b-93f9-72c726ffc8c8","encodingList":[],"isDimensionColumn":true,"scale":-1,"precision":-1,"schemaOrdinal":7,"numberOfChild":0,"invisible":false,"isSortColumn":false,"aggFunction":"","timeSeriesFunction":"","isLocalDictColumn":true},{"dataType":

      {"id":2,"precedenceOrder":2,"name":"TIMESTAMP","sizeInBytes":-1}

      ,"columnName":"projectjoindate","columnUniqueId":"0f364fa9-5b97-476e-88c0-87e8a17bd6db","columnReferenceId":"0f364fa9-5b97-476e-88c0-87e8a17bd6db","encodingList":[],"isDimensionColumn":true,"scale":-1,"precision":-1,"schemaOrdinal":9,"numberOfChild":0,"invisible":false,"isSortColumn":false,"aggFunction":"","timeSeriesFunction":"","isLocalDictColumn":false},{"dataType":

      {"id":2,"precedenceOrder":2,"name":"TIMESTAMP","sizeInBytes":-1}

      ,"columnName":"projectenddate","columnUniqueId":"854de2a9-b251-49b4-a1f1-1bf00d803379","columnReferenceId":"854de2a9-b251-49b4-a1f1-1bf00d803379","encodingList":[],"isDimensionColumn":true,"scale":-1,"precision":-1,"schemaOrdinal":10,"numberOfChild":0,"invisible":false,"isSortColumn":false,"aggFunction":"","timeSeriesFunction":"","isLocalDictColumn":false},{"dataType":

      {"id":5,"precedenceOrder":3,"name":"INT","sizeInBytes":4}

      ,"columnName":"empno","columnUniqueId":"f98b494f-f287-48e6-a085-8cf552b34ce3","columnReferenceId":"f98b494f-f287-48e6-a085-8cf552b34ce3","encodingList":[],"isDimensionColumn":false,"scale":-1,"precision":-1,"schemaOrdinal":0,"numberOfChild":0,"invisible":false,"isSortColumn":false,"aggFunction":"","timeSeriesFunction":"","isLocalDictColumn":false},{"dataType":

      {"id":5,"precedenceOrder":3,"name":"INT","sizeInBytes":4}

      ,"columnName":"workgroupcategory","columnUniqueId":"99f787ae-6f5b-44bb-b134-24b9f6c1e977","columnReferenceId":"99f787ae-6f5b-44bb-b134-24b9f6c1e977","encodingList":[],"isDimensionColumn":false,"scale":-1,"precision":-1,"schemaOrdinal":4,"numberOfChild":0,"invisible":false,"isSortColumn":false,"aggFunction":"","timeSeriesFunction":"","isL',carbonSchema1 'ocalDictColumn":false},{"dataType":

      {"id":5,"precedenceOrder":3,"name":"INT","sizeInBytes":4}

      ,"columnName":"deptno","columnUniqueId":"8b008270-eb13-4ef2-bd51-60f00ab1ec71","columnReferenceId":"8b008270-eb13-4ef2-bd51-60f00ab1ec71","encodingList":[],"isDimensionColumn":false,"scale":-1,"precision":-1,"schemaOrdinal":6,"numberOfChild":0,"invisible":false,"isSortColumn":false,"aggFunction":"","timeSeriesFunction":"","isLocalDictColumn":false},{"dataType":

      {"id":5,"precedenceOrder":3,"name":"INT","sizeInBytes":4}

      ,"columnName":"projectcode","columnUniqueId":"d95fe8d3-4de8-4ec8-bd4c-e3855f6f979f","columnReferenceId":"d95fe8d3-4de8-4ec8-bd4c-e3855f6f979f","encodingList":[],"isDimensionColumn":false,"scale":-1,"precision":-1,"schemaOrdinal":8,"numberOfChild":0,"invisible":false,"isSortColumn":false,"aggFunction":"","timeSeriesFunction":"","isLocalDictColumn":false},{"dataType":

      {"id":5,"precedenceOrder":3,"name":"INT","sizeInBytes":4}

      ,"columnName":"attendance","columnUniqueId":"3c93c60d-9a78-4943-9a37-ac7c8b4144d3","columnReferenceId":"3c93c60d-9a78-4943-9a37-ac7c8b4144d3","encodingList":[],"isDimensionColumn":false,"scale":-1,"precision":-1,"schemaOrdinal":11,"numberOfChild":0,"invisible":false,"isSortColumn":false,"aggFunction":"","timeSeriesFunction":"","isLocalDictColumn":false},{"dataType":

      {"id":5,"precedenceOrder":3,"name":"INT","sizeInBytes":4}

      ,"columnName":"utilization","columnUniqueId":"9f7beba6-5a80-4a43-abbd-abe25d221638","columnReferenceId":"9f7beba6-5a80-4a43-abbd-abe25d221638","encodingList":[],"isDimensionColumn":false,"scale":-1,"precision":-1,"schemaOrdinal":12,"numberOfChild":0,"invisible":false,"isSortColumn":false,"aggFunction":"","timeSeriesFunction":"","isLocalDictColumn":false},{"dataType":

      {"id":5,"precedenceOrder":3,"name":"INT","sizeInBytes":4}

      ,"columnName":"salary","columnUniqueId":"4c9ef8cb-b26d-4233-a030-6bfd32b18e83","columnReferenceId":"4c9ef8cb-b26d-4233-a030-6bfd32b18e83","encodingList":[],"isDimensionColumn":false,"scale":-1,"precision":-1,"schemaOrdinal":13,"numberOfChild":0,"invisible":false,"isSortColumn":false,"aggFunction":"","timeSeriesFunction":"","isLocalDictColumn":false}],"schemaEvolution":{"schemaEvolutionEntryList":[

      {"timeStamp":1622772105375,"added":[],"removed":[]}

      ,{"timeStamp":1622772105375}]},"tableProperties":{"indexexists":"false","sort_columns":"","comment":"","relatedmvtablesmap":"

      {\"2_1\":[\"datamap_comp_mv\"]}

      ","bad_record_path":"","local_dictionary_enable":"true","indextableexists":"false"}},"lastUpdatedTime":1622772105375,"tablePath":"hdfs://hacluster/user/hive/warehouse/carbon.store/2_1/origintable_mv","isTransactionalTable":true,"hasColumnDrift":false,"identifier":{"tablePath":"hdfs://hacluster/user/hive/warehouse/carbon.store/2_1/origintable_mv","carbonTableIdentifier":{"tableUniqueName":"2_1_origintable_mv","databaseName":"2_1","tableName":"origintable_mv","tableId":"5b9f23bf-c08a-49bf-9b33-6f1a397014e5"}},"isSchemaModified":false}')

      == Carbon Parser: org.apache.spark.sql.parser.CarbonExtensionSpark2SqlParser ==

      Attachments

        Activity

          People

            Unassigned Unassigned
            Shreelekhya SHREELEKHYA GAMPA
            Votes:
            0 Vote for this issue
            Watchers:
            1 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved:

              Time Tracking

                Estimated:
                Original Estimate - Not Specified
                Not Specified
                Remaining:
                Remaining Estimate - 0h
                0h
                Logged:
                Time Spent - 2h 50m
                2h 50m