[root@inspur1 spark-2.3.3-bin-hadoop2.7]# ./bin/spark-submit --class org.apache.spark.examples.SparkPi \ > --master yarn \ > --deploy-mode client \ > --driver-memory 512M \ > --executor-memory 512M \ > --jars yarn/spark-2.3.3-yarn-shuffle.jar\ > --executor-cores 1 \ > examples/jars/spark-examples*.jar \ > 2 19/04/18 13:51:03 INFO SparkContext: Running Spark version 2.3.3 19/04/18 13:51:03 INFO SparkContext: Submitted application: Spark Pi 19/04/18 13:51:03 INFO SecurityManager: Changing view acls to: root 19/04/18 13:51:03 INFO SecurityManager: Changing modify acls to: root 19/04/18 13:51:03 INFO SecurityManager: Changing view acls groups to: 19/04/18 13:51:03 INFO SecurityManager: Changing modify acls groups to: 19/04/18 13:51:03 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(root); groups with view permissions: Set(); users with modify permissions: Set(root); groups with modify permissions: Set() 19/04/18 13:51:03 INFO Utils: Successfully started service 'sparkDriver' on port 44558. 19/04/18 13:51:03 INFO SparkEnv: Registering MapOutputTracker 19/04/18 13:51:03 INFO SparkEnv: Registering BlockManagerMaster 19/04/18 13:51:03 INFO BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information 19/04/18 13:51:03 INFO BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up 19/04/18 13:51:03 INFO DiskBlockManager: Created local directory at /tmp/blockmgr-303049fc-5d70-4770-b17f-5476ca418d79 19/04/18 13:51:03 INFO MemoryStore: MemoryStore started with capacity 93.3 MB 19/04/18 13:51:03 INFO SparkEnv: Registering OutputCommitCoordinator 19/04/18 13:51:04 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041. 19/04/18 13:51:04 INFO Utils: Successfully started service 'SparkUI' on port 4041. 19/04/18 13:51:04 INFO SparkUI: Bound SparkUI to 0.0.0.0, and started at http://inspur1:4041 19/04/18 13:51:04 INFO SparkContext: Added JAR file:/usr/local/install/spark-2.3.3-bin-hadoop2.7/examples/jars/spark-examples_2.11-2.3.3.jar at spark://inspur1:44558/jars/spark-examples_2.11-2.3.3.jar with timestamp 1555566664265 19/04/18 13:51:04 INFO RMProxy: Connecting to ResourceManager at inspur1/10.166.16.141:8050 19/04/18 13:51:05 ERROR SparkContext: Error initializing SparkContext. org.apache.commons.lang.NotImplementedException: Code is not implemented at org.apache.hadoop.yarn.server.router.clientrm.FederationClientInterceptor.getClusterMetrics(FederationClientInterceptor.java:573) at org.apache.hadoop.yarn.server.router.clientrm.RouterClientRMService.getClusterMetrics(RouterClientRMService.java:230) at org.apache.hadoop.yarn.api.impl.pb.service.ApplicationClientProtocolPBServiceImpl.getClusterMetrics(ApplicationClientProtocolPBServiceImpl.java:248) at org.apache.hadoop.yarn.proto.ApplicationClientProtocol$ApplicationClientProtocolService$2.callBlockingMethod(ApplicationClientProtocol.java:569) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:523) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:991) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:872) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:818) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1729) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2678) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateException(RPCUtil.java:53) at org.apache.hadoop.yarn.ipc.RPCUtil.unwrapAndThrowException(RPCUtil.java:107) at org.apache.hadoop.yarn.api.impl.pb.client.ApplicationClientProtocolPBClientImpl.getClusterMetrics(ApplicationClientProtocolPBClientImpl.java:209) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:191) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) at com.sun.proxy.$Proxy16.getClusterMetrics(Unknown Source) at org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.getYarnClusterMetrics(YarnClientImpl.java:487) at org.apache.spark.deploy.yarn.Client$$anonfun$submitApplication$1.apply(Client.scala:155) at org.apache.spark.deploy.yarn.Client$$anonfun$submitApplication$1.apply(Client.scala:155) at org.apache.spark.internal.Logging$class.logInfo(Logging.scala:54) at org.apache.spark.deploy.yarn.Client.logInfo(Client.scala:59) at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:154) at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.start(YarnClientSchedulerBackend.scala:57) at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:164) at org.apache.spark.SparkContext.(SparkContext.scala:500) at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2493) at org.apache.spark.sql.SparkSession$Builder$$anonfun$7.apply(SparkSession.scala:934) at org.apache.spark.sql.SparkSession$Builder$$anonfun$7.apply(SparkSession.scala:925) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:925) at org.apache.spark.examples.SparkPi$.main(SparkPi.scala:31) at org.apache.spark.examples.SparkPi.main(SparkPi.scala) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52) at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:894) at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:198) at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:228) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:137) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) 19/04/18 13:51:05 INFO SparkUI: Stopped Spark web UI at http://inspur1:4041 19/04/18 13:51:05 WARN YarnSchedulerBackend$YarnSchedulerEndpoint: Attempted to request executors before the AM has registered! 19/04/18 13:51:05 INFO YarnClientSchedulerBackend: Stopped 19/04/18 13:51:05 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped! 19/04/18 13:51:05 INFO MemoryStore: MemoryStore cleared 19/04/18 13:51:05 INFO BlockManager: BlockManager stopped 19/04/18 13:51:05 INFO BlockManagerMaster: BlockManagerMaster stopped 19/04/18 13:51:05 WARN MetricsSystem: Stopping a MetricsSystem that is not running 19/04/18 13:51:05 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped! 19/04/18 13:51:05 INFO SparkContext: Successfully stopped SparkContext Exception in thread "main" org.apache.commons.lang.NotImplementedException: Code is not implemented at org.apache.hadoop.yarn.server.router.clientrm.FederationClientInterceptor.getClusterMetrics(FederationClientInterceptor.java:573) at org.apache.hadoop.yarn.server.router.clientrm.RouterClientRMService.getClusterMetrics(RouterClientRMService.java:230) at org.apache.hadoop.yarn.api.impl.pb.service.ApplicationClientProtocolPBServiceImpl.getClusterMetrics(ApplicationClientProtocolPBServiceImpl.java:248) at org.apache.hadoop.yarn.proto.ApplicationClientProtocol$ApplicationClientProtocolService$2.callBlockingMethod(ApplicationClientProtocol.java:569) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:523) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:991) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:872) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:818) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1729) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2678) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateException(RPCUtil.java:53) at org.apache.hadoop.yarn.ipc.RPCUtil.unwrapAndThrowException(RPCUtil.java:107) at org.apache.hadoop.yarn.api.impl.pb.client.ApplicationClientProtocolPBClientImpl.getClusterMetrics(ApplicationClientProtocolPBClientImpl.java:209) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:191) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) at com.sun.proxy.$Proxy16.getClusterMetrics(Unknown Source) at org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.getYarnClusterMetrics(YarnClientImpl.java:487) at org.apache.spark.deploy.yarn.Client$$anonfun$submitApplication$1.apply(Client.scala:155) at org.apache.spark.deploy.yarn.Client$$anonfun$submitApplication$1.apply(Client.scala:155) at org.apache.spark.internal.Logging$class.logInfo(Logging.scala:54) at org.apache.spark.deploy.yarn.Client.logInfo(Client.scala:59) at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:154) at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.start(YarnClientSchedulerBackend.scala:57) at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:164) at org.apache.spark.SparkContext.(SparkContext.scala:500) at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2493) at org.apache.spark.sql.SparkSession$Builder$$anonfun$7.apply(SparkSession.scala:934) at org.apache.spark.sql.SparkSession$Builder$$anonfun$7.apply(SparkSession.scala:925) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:925) at org.apache.spark.examples.SparkPi$.main(SparkPi.scala:31) at org.apache.spark.examples.SparkPi.main(SparkPi.scala) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52) at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:894) at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:198) at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:228) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:137) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) 19/04/18 13:51:05 INFO ShutdownHookManager: Shutdown hook called 19/04/18 13:51:05 INFO ShutdownHookManager: Deleting directory /tmp/spark-83a9593f-d42f-4b7f-b48a-7270e6633455 19/04/18 13:51:05 INFO ShutdownHookManager: Deleting directory /tmp/spark-25780206-ed4e-48bb-b36b-764d6fbf7676