AnsweredAssumed Answered

org.apache.hadoop.mapred.InvalidInputException: Input path does not exist: maprfs?

Question asked by anilmapr on Jun 21, 2016
Latest reply on Jun 21, 2016 by anilmapr

Hi, I tried executing the word count in spark. I switched into Mapr cli and created the file and executed the following command. its showing input path doesnt exist.

How to place the files in Mapr file system in this scenario to solve the error?

 

scala> val inputfile1 = sc.textFile("/home/mapr/Simple.txt")

inputfile1: org.apache.spark.rdd.RDD[String] = MapPartitionsRDD[7] at textFile a                                                                                                                                                             t <console>:21

 

 

scala> val counts = inputfile1.flatMap(line => line.split(" ")).map(word => (wor                                                                                                                                                             d, 1)).reduceByKey(_+_);

org.apache.hadoop.mapred.InvalidInputException: Input path does not exist: maprf                                                                                                                                                             s:/home/mapr/Simple.txt

        at org.apache.hadoop.mapred.FileInputFormat.singleThreadedListStatus(Fil                                                                                                                                                             eInputFormat.java:289)

        at org.apache.hadoop.mapred.FileInputFormat.listStatus(FileInputFormat.j                                                                                                                                                             ava:229)

        at org.apache.hadoop.mapred.FileInputFormat.getSplits(FileInputFormat.ja                                                                                                                                                             va:317)

        at org.apache.spark.rdd.HadoopRDD.getPartitions(HadoopRDD.scala:207)

        at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)

        at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)

        at scala.Option.getOrElse(Option.scala:120)

        at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)

        at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.                                                                                                                                                             scala:35)

        at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)

        at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)

        at scala.Option.getOrElse(Option.scala:120)

        at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)

        at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.                                                                                                                                                             scala:35)

        at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)

        at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)

        at scala.Option.getOrElse(Option.scala:120)

        at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)

        at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.                                                                                                                                                             scala:35)

        at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)

        at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)

        at scala.Option.getOrElse(Option.scala:120)

        at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)

        at org.apache.spark.Partitioner$.defaultPartitioner(Partitioner.scala:65                                                                                                                                                             )

        at org.apache.spark.rdd.PairRDDFunctions$$anonfun$reduceByKey$3.apply(Pa                                                                                                                                                             irRDDFunctions.scala:290)

        at org.apache.spark.rdd.PairRDDFunctions$$anonfun$reduceByKey$3.apply(Pa                                                                                                                                                             irRDDFunctions.scala:290)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.s                                                                                                                                                             cala:147)

        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.s                                                                                                                                                             cala:108)

        at org.apache.spark.rdd.RDD.withScope(RDD.scala:310)

        at org.apache.spark.rdd.PairRDDFunctions.reduceByKey(PairRDDFunctions.sc                                                                                                                                                             ala:289)

        at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:23)

        at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:28)

        at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:30)

        at $iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:32)

        at $iwC$$iwC$$iwC$$iwC.<init>(<console>:34)

        at $iwC$$iwC$$iwC.<init>(<console>:36)

        at $iwC$$iwC.<init>(<console>:38)

        at $iwC.<init>(<console>:40)

        at <init>(<console>:42)

        at .<init>(<console>:46)

        at .<clinit>(<console>)

        at .<init>(<console>:7)

        at .<clinit>(<console>)

        at $print(<console>)

        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.                                                                                                                                                             java:57)

        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAcces                                                                                                                                                             sorImpl.java:43)

        at java.lang.reflect.Method.invoke(Method.java:606)

        at org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:                                                                                                                                                             1065)

        at org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:                                                                                                                                                             1340)

        at org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:840                                                                                                                                                             )

        at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:871)

        at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:819)

        at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:8                                                                                                                                                             57)

        at org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.sca                                                                                                                                                             la:902)

        at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:814)

        at org.apache.spark.repl.SparkILoop.processLine$1(SparkILoop.scala:657)

        at org.apache.spark.repl.SparkILoop.innerLoop$1(SparkILoop.scala:665)

        at org.apache.spark.repl.SparkILoop.org$apache$spark$repl$SparkILoop$$lo                                                                                                                                                             op(SparkILoop.scala:670)

        at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$Spark                                                                                                                                                             ILoop$$process$1.apply$mcZ$sp(SparkILoop.scala:997)

        at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$Spark                                                                                                                                                             ILoop$$process$1.apply(SparkILoop.scala:945)

        at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$Spark                                                                                                                                                             ILoop$$process$1.apply(SparkILoop.scala:945)

        at scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClass                                                                                                                                                             Loader.scala:135)

        at org.apache.spark.repl.SparkILoop.org$apache$spark$repl$SparkILoop$$pr                                                                                                                                                             ocess(SparkILoop.scala:945)

        at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:1059)

        at org.apache.spark.repl.Main$.main(Main.scala:31)

        at org.apache.spark.repl.Main.main(Main.scala)

        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.                                                                                                                                                             java:57)

        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAcces                                                                                                                                                             sorImpl.java:43)

        at java.lang.reflect.Method.invoke(Method.java:606)

        at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSub                                                                                                                                                             mit$$runMain(SparkSubmit.scala:674)

        at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:18                                                                                                                                                             0)

        at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205)

        at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:120)

        at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)

Outcomes