Uploaded image for project: 'Spark'
  1. Spark
  2. SPARK-10709

When loading a json dataset as a data frame, if the input path is wrong, the error message is very confusing

    XMLWordPrintableJSON

    Details

    • Type: Improvement
    • Status: Resolved
    • Priority: Minor
    • Resolution: Duplicate
    • Affects Version/s: None
    • Fix Version/s: None
    • Component/s: SQL
    • Labels:
      None

      Description

      If you do something like sqlContext.read.json("a wrong path"), when we actually read data, the error message is

      java.io.IOException: No input paths specified in job
      	at org.apache.hadoop.mapred.FileInputFormat.listStatus(FileInputFormat.java:198)
      	at org.apache.hadoop.mapred.FileInputFormat.getSplits(FileInputFormat.java:270)
      	at org.apache.spark.rdd.HadoopRDD.getPartitions(HadoopRDD.scala:207)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)
      	at scala.Option.getOrElse(Option.scala:120)
      	at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)
      	at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)
      	at scala.Option.getOrElse(Option.scala:120)
      	at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)
      	at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)
      	at scala.Option.getOrElse(Option.scala:120)
      	at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)
      	at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)
      	at scala.Option.getOrElse(Option.scala:120)
      	at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)
      	at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)
      	at scala.Option.getOrElse(Option.scala:120)
      	at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)
      	at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)
      	at scala.Option.getOrElse(Option.scala:120)
      	at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)
      	at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)
      	at scala.Option.getOrElse(Option.scala:120)
      	at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)
      	at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)
      	at scala.Option.getOrElse(Option.scala:120)
      	at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)
      	at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)
      	at scala.Option.getOrElse(Option.scala:120)
      	at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)
      	at org.apache.spark.ShuffleDependency.<init>(Dependency.scala:91)
      	at org.apache.spark.sql.execution.ShuffledRowRDD.getDependencies(ShuffledRowRDD.scala:59)
      	at org.apache.spark.rdd.RDD$$anonfun$dependencies$2.apply(RDD.scala:226)
      	at org.apache.spark.rdd.RDD$$anonfun$dependencies$2.apply(RDD.scala:224)
      	at scala.Option.getOrElse(Option.scala:120)
      	at org.apache.spark.rdd.RDD.dependencies(RDD.scala:224)
      	at org.apache.spark.scheduler.DAGScheduler.visit$2(DAGScheduler.scala:427)
      	at org.apache.spark.scheduler.DAGScheduler.getAncestorShuffleDependencies(DAGScheduler.scala:442)
      	at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$getShuffleMapStage(DAGScheduler.scala:301)
      	at org.apache.spark.scheduler.DAGScheduler$$anonfun$visit$1$1.apply(DAGScheduler.scala:403)
      	at org.apache.spark.scheduler.DAGScheduler$$anonfun$visit$1$1.apply(DAGScheduler.scala:400)
      	at scala.collection.immutable.List.foreach(List.scala:318)
      	at org.apache.spark.scheduler.DAGScheduler.visit$1(DAGScheduler.scala:400)
      	at org.apache.spark.scheduler.DAGScheduler.getParentStages(DAGScheduler.scala:412)
      	at org.apache.spark.scheduler.DAGScheduler.getParentStagesAndId(DAGScheduler.scala:315)
      	at org.apache.spark.scheduler.DAGScheduler.newResultStage(DAGScheduler.scala:350)
      	at org.apache.spark.scheduler.DAGScheduler.handleJobSubmitted(DAGScheduler.scala:846)
      	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1637)
      	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1629)
      	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1618)
      	at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
      java.io.IOException: No input paths specified in job
      	at org.apache.hadoop.mapred.FileInputFormat.listStatus(FileInputFormat.java:198)
      	at org.apache.hadoop.mapred.FileInputFormat.getSplits(FileInputFormat.java:270)
      	at org.apache.spark.rdd.HadoopRDD.getPartitions(HadoopRDD.scala:207)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)
      	at scala.Option.getOrElse(Option.scala:120)
      	at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)
      	at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)
      	at scala.Option.getOrElse(Option.scala:120)
      	at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)
      	at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)
      	at scala.Option.getOrElse(Option.scala:120)
      	at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)
      	at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)
      	at scala.Option.getOrElse(Option.scala:120)
      	at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)
      	at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)
      	at scala.Option.getOrElse(Option.scala:120)
      	at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)
      	at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)
      	at scala.Option.getOrElse(Option.scala:120)
      	at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)
      	at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)
      	at scala.Option.getOrElse(Option.scala:120)
      	at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)
      	at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)
      	at scala.Option.getOrElse(Option.scala:120)
      	at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)
      	at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)
      	at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)
      	at scala.Option.getOrElse(Option.scala:120)
      	at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)
      	at org.apache.spark.ShuffleDependency.<init>(Dependency.scala:91)
      	at org.apache.spark.sql.execution.ShuffledRowRDD.getDependencies(ShuffledRowRDD.scala:59)
      	at org.apache.spark.rdd.RDD$$anonfun$dependencies$2.apply(RDD.scala:226)
      	at org.apache.spark.rdd.RDD$$anonfun$dependencies$2.apply(RDD.scala:224)
      	at scala.Option.getOrElse(Option.scala:120)
      	at org.apache.spark.rdd.RDD.dependencies(RDD.scala:224)
      	at org.apache.spark.scheduler.DAGScheduler.visit$2(DAGScheduler.scala:427)
      	at org.apache.spark.scheduler.DAGScheduler.getAncestorShuffleDependencies(DAGScheduler.scala:442)
      	at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$getShuffleMapStage(DAGScheduler.scala:301)
      	at org.apache.spark.scheduler.DAGScheduler$$anonfun$visit$1$1.apply(DAGScheduler.scala:403)
      	at org.apache.spark.scheduler.DAGScheduler$$anonfun$visit$1$1.apply(DAGScheduler.scala:400)
      	at scala.collection.immutable.List.foreach(List.scala:318)
      	at org.apache.spark.scheduler.DAGScheduler.visit$1(DAGScheduler.scala:400)
      	at org.apache.spark.scheduler.DAGScheduler.getParentStages(DAGScheduler.scala:412)
      	at org.apache.spark.scheduler.DAGScheduler.getParentStagesAndId(DAGScheduler.scala:315)
      	at org.apache.spark.scheduler.DAGScheduler.newResultStage(DAGScheduler.scala:350)
      	at org.apache.spark.scheduler.DAGScheduler.handleJobSubmitted(DAGScheduler.scala:846)
      	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1637)
      	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1629)
      	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1618)
      	at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
      	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:630)
      	at org.apache.spark.SparkContext.runJob(SparkContext.scala:1823)
      	at org.apache.spark.SparkContext.runJob(SparkContext.scala:1943)
      	at org.apache.spark.rdd.RDD$$anonfun$reduce$1.apply(RDD.scala:997)
      	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:150)
      	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:111)
      	at org.apache.spark.rdd.RDD.withScope(RDD.scala:306)
      	at org.apache.spark.rdd.RDD.reduce(RDD.scala:979)
      	at org.apache.spark.rdd.RDD$$anonfun$takeOrdered$1.apply(RDD.scala:1360)
      	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:150)
      	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:111)
      	at org.apache.spark.rdd.RDD.withScope(RDD.scala:306)
      	at org.apache.spark.rdd.RDD.takeOrdered(RDD.scala:1347)
      	at org.apache.spark.sql.execution.TakeOrderedAndProject.collectData(basicOperators.scala:257)
      	at org.apache.spark.sql.execution.TakeOrderedAndProject.executeCollect(basicOperators.scala:263)
      	at org.apache.spark.sql.DataFrame$$anonfun$collect$1.apply(DataFrame.scala:1399)
      	at org.apache.spark.sql.DataFrame$$anonfun$collect$1.apply(DataFrame.scala:1399)
      	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:56)
      	at org.apache.spark.sql.DataFrame.withNewExecutionId(DataFrame.scala:1917)
      	at org.apache.spark.sql.DataFrame.collect(DataFrame.scala:1398)
      	at org.apache.spark.sql.DataFrame.head(DataFrame.scala:1328)
      	at org.apache.spark.sql.DataFrame.take(DataFrame.scala:1391)
      	at org.apache.spark.sql.DataFrame.showString(DataFrame.scala:178)
      	at org.apache.spark.sql.DataFrame.show(DataFrame.scala:402)
      	at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:37)
      	at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:42)
      	at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:44)
      	at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:46)
      	at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:48)
      	at $iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:50)
      	at $iwC$$iwC$$iwC$$iwC.<init>(<console>:52)
      	at $iwC$$iwC$$iwC.<init>(<console>:54)
      	at $iwC$$iwC.<init>(<console>:56)
      	at $iwC.<init>(<console>:58)
      	at <init>(<console>:60)
      	at .<init>(<console>:64)
      	at .<clinit>(<console>)
      	at .<init>(<console>:7)
      	at .<clinit>(<console>)
      	at $print(<console>)
      	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
      	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
      	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      	at java.lang.reflect.Method.invoke(Method.java:606)
      	at org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:1065)
      	at org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1340)
      	at org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:840)
      	at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:871)
      	at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:819)
      	at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:857)
      	at org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:902)
      	at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:814)
      	at org.apache.spark.repl.SparkILoop.processLine$1(SparkILoop.scala:657)
      	at org.apache.spark.repl.SparkILoop.innerLoop$1(SparkILoop.scala:665)
      	at org.apache.spark.repl.SparkILoop.org$apache$spark$repl$SparkILoop$$loop(SparkILoop.scala:670)
      	at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply$mcZ$sp(SparkILoop.scala:997)
      	at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945)
      	at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945)
      	at scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:135)
      	at org.apache.spark.repl.SparkILoop.org$apache$spark$repl$SparkILoop$$process(SparkILoop.scala:945)
      	at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:1059)
      	at org.apache.spark.repl.Main$.main(Main.scala:31)
      	at org.apache.spark.repl.Main.main(Main.scala)
      

      This message is very confusing.

        Attachments

          Issue Links

            Activity

              People

              • Assignee:
                Unassigned
                Reporter:
                yhuai Yin Huai
              • Votes:
                0 Vote for this issue
                Watchers:
                6 Start watching this issue

                Dates

                • Created:
                  Updated:
                  Resolved: