[CARBONDATA-3469] CarbonData with 2.3.2 can not run on CDH spark 2.4

• Type: Bug
• Status: Open
• Priority: Major
• Resolution: Unresolved
• Affects Version/s: 1.5.3
• Component/s: spark-integration
• Assignee: Unassigned
• Reporter: wxmimperio
• Created: 12/Jul/19 09:44
• Updated: 07/Nov/19 07:47

Description

spark2-shell --jars apache-carbondata-1.5.3-bin-spark2.3.2-hadoop2.7.2.jar

 

java.lang.NoSuchMethodError: org.apache.spark.sql.internal.SharedState.externalCatalog()Lorg/apache/spark/sql/catalyst/catalog/ExternalCatalog;

scala> carbon.sql(
| s"""
| | CREATE TABLE IF NOT EXISTS test_table(
| | id string,
| | name string,
| | city string,
| | age Int)
| | STORED AS carbondata
| """.stripMargin)
java.lang.NoSuchMethodError: org.apache.spark.sql.internal.SharedState.externalCatalog()Lorg/apache/spark/sql/catalyst/catalog/ExternalCatalog;
at org.apache.spark.sql.hive.CarbonSessionStateBuilder.externalCatalog(CarbonSessionState.scala:227)
at org.apache.spark.sql.hive.CarbonSessionStateBuilder.catalog$lzycompute(CarbonSessionState.scala:214)
at org.apache.spark.sql.hive.CarbonSessionStateBuilder.catalog(CarbonSessionState.scala:212)
at org.apache.spark.sql.hive.CarbonSessionStateBuilder.catalog(CarbonSessionState.scala:191)
at org.apache.spark.sql.internal.BaseSessionStateBuilder$$anonfun$build$1.apply(BaseSessionStateBuilder.scala:291)
at org.apache.spark.sql.internal.BaseSessionStateBuilder$$anonfun$build$1.apply(BaseSessionStateBuilder.scala:291)
at org.apache.spark.sql.internal.SessionState.catalog$lzycompute(SessionState.scala:77)
at org.apache.spark.sql.internal.SessionState.catalog(SessionState.scala:77)
at org.apache.spark.sql.CarbonEnv$.getInstance(CarbonEnv.scala:135)
at org.apache.spark.sql.CarbonSession$.updateSessionInfoToCurrentThread(CarbonSession.scala:326)
at org.apache.spark.sql.parser.CarbonSparkSqlParser.parsePlan(CarbonSparkSqlParser.scala:47)
at org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:125)
at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:88)
... 59 elided