AnsweredAssumed Answered

spark-submit with python file fails

Question asked by cdeterman on Aug 30, 2016
Latest reply on Feb 22, 2017 by dobbyssock87


I am working on Lesson 3 of DEV 360.  In that lesson it is recommended to test completed files via spark-submit.  However, even with the provided python solution I am encountering an error.  I run the following line and get the output below.  I can start a pyspark session interactively but this spark-submit component is failing.

 

/opt/mapr/spark/spark-1.5.2/bin/spark-submit AuctionsApp.py --master local

 

16/08/30 10:00:26 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable

16/08/30 10:00:28 WARN MetricsSystem: Using default name DAGScheduler for source because spark.app.id is not set.

16/08/30 10:00:28 WARN AppClient$ClientEndpoint: Could not connect to maprdemo:7077: akka.remote.EndpointAssociationException: Association failed with [akka.tcp://sparkMaster@maprdemo:7077]

16/08/30 10:00:29 WARN ReliableDeliverySupervisor: Association with remote system [akka.tcp://sparkMaster@maprdemo:7077] has failed, address is now gated for [5000] ms. Reason: [Association failed with [akka.tcp://sparkMaster@maprdemo:7077]] Caused by: [Connection refused: maprdemo/10.0.2.15:7077]

16/08/30 10:00:29 WARN AppClient$ClientEndpoint: Failed to connect to master maprdemo:7077

akka.actor.ActorNotFound: Actor not found for: ActorSelection[Anchor(akka.tcp://sparkMaster@maprdemo:7077/), Path(/user/Master)]

        at akka.actor.ActorSelection$$anonfun$resolveOne$1.apply(ActorSelection.scala:65)

        at akka.actor.ActorSelection$$anonfun$resolveOne$1.apply(ActorSelection.scala:63)

        at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:32)

        at akka.dispatch.BatchingExecutor$AbstractBatch.processBatch(BatchingExecutor.scala:55)

        at akka.dispatch.BatchingExecutor$Batch.run(BatchingExecutor.scala:73)

        at akka.dispatch.ExecutionContexts$sameThreadExecutionContext$.unbatchedExecute(Future.scala:74)

        at akka.dispatch.BatchingExecutor$class.execute(BatchingExecutor.scala:120)

        at akka.dispatch.ExecutionContexts$sameThreadExecutionContext$.execute(Future.scala:73)

        at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:40)

        at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:248)

        at akka.pattern.PromiseActorRef.$bang(AskSupport.scala:266)

        at akka.actor.EmptyLocalActorRef.specialHandle(ActorRef.scala:533)

        at akka.actor.DeadLetterActorRef.specialHandle(ActorRef.scala:569)

        at akka.actor.DeadLetterActorRef.$bang(ActorRef.scala:559)

        at akka.remote.RemoteActorRefProvider$RemoteDeadLetterActorRef.$bang(RemoteActorRefProvider.scala:87)

        at akka.remote.EndpointWriter.postStop(Endpoint.scala:557)

        at akka.actor.Actor$class.aroundPostStop(Actor.scala:477)

        at akka.remote.EndpointActor.aroundPostStop(Endpoint.scala:411)

        at akka.actor.dungeon.FaultHandling$class.akka$actor$dungeon$FaultHandling$$finishTerminate(FaultHandling.scala:210)

        at akka.actor.dungeon.FaultHandling$class.terminate(FaultHandling.scala:172)

        at akka.actor.ActorCell.terminate(ActorCell.scala:369)

        at akka.actor.ActorCell.invokeAll$1(ActorCell.scala:462)

        at akka.actor.ActorCell.systemInvoke(ActorCell.scala:478)

        at akka.dispatch.Mailbox.processAllSystemMessages(Mailbox.scala:263)

        at akka.dispatch.Mailbox.run(Mailbox.scala:219)

        at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397)

        at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)

        at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)

        at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)

        at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)

16/08/30 10:00:48 WARN AppClient$ClientEndpoint: Could not connect to maprdemo:7077: akka.remote.EndpointAssociationException: Association failed with [akka.tcp://sparkMaster@maprdemo:7077]

16/08/30 10:00:48 WARN AppClient$ClientEndpoint: Failed to connect to master maprdemo:7077

akka.actor.ActorNotFound: Actor not found for: ActorSelection[Anchor(akka.tcp://sparkMaster@maprdemo:7077/), Path(/user/Master)]

        at akka.actor.ActorSelection$$anonfun$resolveOne$1.apply(ActorSelection.scala:65)

        at akka.actor.ActorSelection$$anonfun$resolveOne$1.apply(ActorSelection.scala:63)

        at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:32)

        at akka.dispatch.BatchingExecutor$AbstractBatch.processBatch(BatchingExecutor.scala:55)

        at akka.dispatch.BatchingExecutor$Batch.run(BatchingExecutor.scala:73)

        at akka.dispatch.ExecutionContexts$sameThreadExecutionContext$.unbatchedExecute(Future.scala:74)

        at akka.dispatch.BatchingExecutor$class.execute(BatchingExecutor.scala:120)

        at akka.dispatch.ExecutionContexts$sameThreadExecutionContext$.execute(Future.scala:73)

        at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:40)

        at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:248)

        at akka.pattern.PromiseActorRef.$bang(AskSupport.scala:266)

        at akka.actor.EmptyLocalActorRef.specialHandle(ActorRef.scala:533)

        at akka.actor.DeadLetterActorRef.specialHandle(ActorRef.scala:569)

        at akka.actor.DeadLetterActorRef.$bang(ActorRef.scala:559)

        at akka.remote.RemoteActorRefProvider$RemoteDeadLetterActorRef.$bang(RemoteActorRefProvider.scala:87)

        at akka.remote.EndpointWriter.postStop(Endpoint.scala:557)

        at akka.actor.Actor$class.aroundPostStop(Actor.scala:477)

        at akka.remote.EndpointActor.aroundPostStop(Endpoint.scala:411)

        at akka.actor.dungeon.FaultHandling$class.akka$actor$dungeon$FaultHandling$$finishTerminate(FaultHandling.scala:210)

        at akka.actor.dungeon.FaultHandling$class.terminate(FaultHandling.scala:172)

        at akka.actor.ActorCell.terminate(ActorCell.scala:369)

        at akka.actor.ActorCell.invokeAll$1(ActorCell.scala:462)

        at akka.actor.ActorCell.systemInvoke(ActorCell.scala:478)

        at akka.dispatch.Mailbox.processAllSystemMessages(Mailbox.scala:263)

        at akka.dispatch.Mailbox.run(Mailbox.scala:219)

        at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397)

        at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)

        at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)

        at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)

        at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)

16/08/30 10:00:48 WARN ReliableDeliverySupervisor: Association with remote system [akka.tcp://sparkMaster@maprdemo:7077] has failed, address is now gated for [5000] ms. Reason: [Association failed with [akka.tcp://sparkMaster@maprdemo:7077]] Caused by: [Connection refused: maprdemo/10.0.2.15:7077]

16/08/30 10:01:08 ERROR SparkUncaughtExceptionHandler: Uncaught exception in thread Thread[appclient-registration-retry-thread,5,main]

java.util.concurrent.RejectedExecutionException: Task java.util.concurrent.FutureTask@1a9ddcb7 rejected from java.util.concurrent.ThreadPoolExecutor@72058007[Running, pool size = 1, active threads = 0, queued tasks = 0, completed tasks = 2]

        at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)

        at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)

        at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)

        at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:110)

        at org.apache.spark.deploy.client.AppClient$ClientEndpoint$$anonfun$tryRegisterAllMasters$1.apply(AppClient.scala:96)

        at org.apache.spark.deploy.client.AppClient$ClientEndpoint$$anonfun$tryRegisterAllMasters$1.apply(AppClient.scala:95)

        at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)

        at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)

        at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)

        at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:108)

        at scala.collection.TraversableLike$class.map(TraversableLike.scala:244)

        at scala.collection.mutable.ArrayOps$ofRef.map(ArrayOps.scala:108)

        at org.apache.spark.deploy.client.AppClient$ClientEndpoint.tryRegisterAllMasters(AppClient.scala:95)

        at org.apache.spark.deploy.client.AppClient$ClientEndpoint.org$apache$spark$deploy$client$AppClient$ClientEndpoint$$registerWithMaster(AppClient.scala:121)

        at org.apache.spark.deploy.client.AppClient$ClientEndpoint$$anon$2$$anonfun$run$1.apply$mcV$sp(AppClient.scala:132)

        at org.apache.spark.util.Utils$.tryOrExit(Utils.scala:1119)

        at org.apache.spark.deploy.client.AppClient$ClientEndpoint$$anon$2.run(AppClient.scala:124)

        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)

        at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)

        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)

        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)

        at java.lang.Thread.run(Thread.java:745)

Outcomes