XMLWordPrintableJSON

    Details

    • Type: Sub-task
    • Status: Resolved
    • Priority: Blocker
    • Resolution: Fixed
    • Affects Version/s: 1.2.0, 1.3.0
    • Fix Version/s: 1.4.0
    • Component/s: SQL
    • Labels:
      None
    • Environment:

      Spark cluster: EC2 m1.small + Spark 1.2.0
      Cassandra cluster: EC2 m3.xlarge + Cassandra 2.1.2

    • Target Version/s:
    • Sprint:
      Spark 1.5 doc/QA sprint

      Description

      My cassandra table task_trace has a field sm.result which contains dot in the name. So SQL tried to look up sm instead of full name 'sm.result'.

      Here is my code:

      scala> import org.apache.spark.sql.cassandra.CassandraSQLContext
      scala> val cc = new CassandraSQLContext(sc)
      scala> val task_trace = cc.jsonFile("/task_trace.json")
      scala> task_trace.registerTempTable("task_trace")
      scala> cc.setKeyspace("cerberus_data_v4")
      scala> val res = cc.sql("SELECT received_datetime, task_body.cerberus_id, task_body.sm.result FROM task_trace WHERE task_id = 'fff7304e-9984-4b45-b10c-0423a96745ce'")
      res: org.apache.spark.sql.SchemaRDD = 
      SchemaRDD[57] at RDD at SchemaRDD.scala:108
      == Query Plan ==
      == Physical Plan ==
      java.lang.RuntimeException: No such struct field sm in cerberus_batch_id, cerberus_id, couponId, coupon_code, created, description, domain, expires, message_id, neverShowAfter, neverShowBefore, offerTitle, screenshots, sm.result, sm.task, startDate, task_id, url, uuid, validationDateTime, validity
      

      The full schema look like this:

      scala> task_trace.printSchema()
      root
       \|-- received_datetime: long (nullable = true)
       \|-- task_body: struct (nullable = true)
       \|    \|-- cerberus_batch_id: string (nullable = true)
       \|    \|-- cerberus_id: string (nullable = true)
       \|    \|-- couponId: integer (nullable = true)
       \|    \|-- coupon_code: string (nullable = true)
       \|    \|-- created: string (nullable = true)
       \|    \|-- description: string (nullable = true)
       \|    \|-- domain: string (nullable = true)
       \|    \|-- expires: string (nullable = true)
       \|    \|-- message_id: string (nullable = true)
       \|    \|-- neverShowAfter: string (nullable = true)
       \|    \|-- neverShowBefore: string (nullable = true)
       \|    \|-- offerTitle: string (nullable = true)
       \|    \|-- screenshots: array (nullable = true)
       \|    \|    \|-- element: string (containsNull = false)
       \|    \|-- sm.result: struct (nullable = true)
       \|    \|    \|-- cerberus_batch_id: string (nullable = true)
       \|    \|    \|-- cerberus_id: string (nullable = true)
       \|    \|    \|-- code: string (nullable = true)
       \|    \|    \|-- couponId: integer (nullable = true)
       \|    \|    \|-- created: string (nullable = true)
       \|    \|    \|-- description: string (nullable = true)
       \|    \|    \|-- domain: string (nullable = true)
       \|    \|    \|-- expires: string (nullable = true)
       \|    \|    \|-- message_id: string (nullable = true)
       \|    \|    \|-- neverShowAfter: string (nullable = true)
       \|    \|    \|-- neverShowBefore: string (nullable = true)
       \|    \|    \|-- offerTitle: string (nullable = true)
       \|    \|    \|-- result: struct (nullable = true)
       \|    \|    \|    \|-- post: struct (nullable = true)
       \|    \|    \|    \|    \|-- alchemy_out_of_stock: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: boolean (nullable = true)
       \|    \|    \|    \|    \|-- meta: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- None_tx_value: array (nullable = true)
       \|    \|    \|    \|    \|    \|    \|-- element: string (containsNull = false)
       \|    \|    \|    \|    \|    \|-- exceptions: array (nullable = true)
       \|    \|    \|    \|    \|    \|    \|-- element: string (containsNull = false)
       \|    \|    \|    \|    \|    \|-- no_input_value: array (nullable = true)
       \|    \|    \|    \|    \|    \|    \|-- element: string (containsNull = false)
       \|    \|    \|    \|    \|    \|-- not_mapped: array (nullable = true)
       \|    \|    \|    \|    \|    \|    \|-- element: string (containsNull = false)
       \|    \|    \|    \|    \|    \|-- not_transformed: array (nullable = true)
       \|    \|    \|    \|    \|    \|    \|-- element: array (containsNull = false)
       \|    \|    \|    \|    \|    \|    \|    \|-- element: string (containsNull = false)
       \|    \|    \|    \|    \|-- now_price_checkout: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: double (nullable = true)
       \|    \|    \|    \|    \|-- shipping_price: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: double (nullable = true)
       \|    \|    \|    \|    \|-- tax: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: double (nullable = true)
       \|    \|    \|    \|    \|-- total: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: double (nullable = true)
       \|    \|    \|    \|-- pre: struct (nullable = true)
       \|    \|    \|    \|    \|-- alchemy_out_of_stock: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: boolean (nullable = true)
       \|    \|    \|    \|    \|-- meta: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- None_tx_value: array (nullable = true)
       \|    \|    \|    \|    \|    \|    \|-- element: string (containsNull = false)
       \|    \|    \|    \|    \|    \|-- exceptions: array (nullable = true)
       \|    \|    \|    \|    \|    \|    \|-- element: string (containsNull = false)
       \|    \|    \|    \|    \|    \|-- no_input_value: array (nullable = true)
       \|    \|    \|    \|    \|    \|    \|-- element: string (containsNull = false)
       \|    \|    \|    \|    \|    \|-- not_mapped: array (nullable = true)
       \|    \|    \|    \|    \|    \|    \|-- element: string (containsNull = false)
       \|    \|    \|    \|    \|    \|-- not_transformed: array (nullable = true)
       \|    \|    \|    \|    \|    \|    \|-- element: array (containsNull = false)
       \|    \|    \|    \|    \|    \|    \|    \|-- element: string (containsNull = false)
       \|    \|    \|    \|    \|-- now_price_checkout: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: double (nullable = true)
       \|    \|    \|    \|    \|-- shipping_price: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: double (nullable = true)
       \|    \|    \|    \|    \|-- tax: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: double (nullable = true)
       \|    \|    \|    \|    \|-- total: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: double (nullable = true)
       \|    \|    \|-- startDate: string (nullable = true)
       \|    \|    \|-- status: string (nullable = true)
       \|    \|    \|-- task_id: string (nullable = true)
       \|    \|    \|-- url: string (nullable = true)
       \|    \|    \|-- uuid: string (nullable = true)
       \|    \|    \|-- validationDateTime: string (nullable = true)
       \|    \|    \|-- validity: string (nullable = true)
       \|    \|-- sm.task: struct (nullable = true)
       \|    \|    \|-- cerberus_batch_id: string (nullable = true)
       \|    \|    \|-- cerberus_id: string (nullable = true)
       \|    \|    \|-- couponId: integer (nullable = true)
       \|    \|    \|-- coupon_code: string (nullable = true)
       \|    \|    \|-- created: string (nullable = true)
       \|    \|    \|-- description: string (nullable = true)
       \|    \|    \|-- domain: string (nullable = true)
       \|    \|    \|-- expires: string (nullable = true)
       \|    \|    \|-- message_id: string (nullable = true)
       \|    \|    \|-- neverShowAfter: string (nullable = true)
       \|    \|    \|-- neverShowBefore: string (nullable = true)
       \|    \|    \|-- offerTitle: string (nullable = true)
       \|    \|    \|-- result: struct (nullable = true)
       \|    \|    \|    \|-- post: struct (nullable = true)
       \|    \|    \|    \|    \|-- final_price: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: integer (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
       \|    \|    \|    \|    \|-- merchant_exception: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: boolean (nullable = true)
       \|    \|    \|    \|    \|-- now_price_checkout: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
       \|    \|    \|    \|    \|-- orig_price: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: integer (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
       \|    \|    \|    \|    \|-- out_of_stock: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: boolean (nullable = true)
       \|    \|    \|    \|    \|-- sale_price: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: integer (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
       \|    \|    \|    \|    \|-- shipping_price: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
       \|    \|    \|    \|    \|-- tax: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
       \|    \|    \|    \|    \|-- total: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
       \|    \|    \|    \|-- pre: struct (nullable = true)
       \|    \|    \|    \|    \|-- final_price: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: integer (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
       \|    \|    \|    \|    \|-- merchant_exception: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: boolean (nullable = true)
       \|    \|    \|    \|    \|-- now_price_checkout: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
       \|    \|    \|    \|    \|-- orig_price: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: integer (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
       \|    \|    \|    \|    \|-- out_of_stock: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: boolean (nullable = true)
       \|    \|    \|    \|    \|-- sale_price: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: integer (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
       \|    \|    \|    \|    \|-- shipping_price: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
       \|    \|    \|    \|    \|-- tax: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
       \|    \|    \|    \|    \|-- total: struct (nullable = true)
       \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
       \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
       \|    \|    \|-- screenshots: array (nullable = true)
       \|    \|    \|    \|-- element: string (containsNull = false)
       \|    \|    \|-- startDate: string (nullable = true)
       \|    \|    \|-- status: string (nullable = true)
       \|    \|    \|-- task_id: string (nullable = true)
       \|    \|    \|-- url: string (nullable = true)
       \|    \|    \|-- uuid: string (nullable = true)
       \|    \|    \|-- validationDateTime: string (nullable = true)
       \|    \|    \|-- validity: string (nullable = true)
       \|    \|-- startDate: string (nullable = true)
       \|    \|-- task_id: string (nullable = true)
       \|    \|-- url: string (nullable = true)
       \|    \|-- uuid: string (nullable = true)
       \|    \|-- validationDateTime: string (nullable = true)
       \|    \|-- validity: string (nullable = true)
       \|-- task_id: string (nullable = true)
       \|-- task_step: integer (nullable = true)
       \|-- task_type: integer (nullable = true)
      

      I also tried val sqlContext = new org.apache.spark.sql.SQLContext(sc), but it give the same exception.

        Attachments

          Issue Links

            Activity

              People

              • Assignee:
                cloud_fan Wenchen Fan
                Reporter:
                lishu Lishu Liu
              • Votes:
                1 Vote for this issue
                Watchers:
                5 Start watching this issue

                Dates

                • Created:
                  Updated:
                  Resolved: