Details
-
Bug
-
Status: Open
-
Major
-
Resolution: Unresolved
-
3.0.0-alpha2
-
None
-
None
-
Ubuntu Linux 4.4.0-72-generic with java-8-openjdk-amd64 1.8.0_121
Description
When yarn.nodemanager.distributed-scheduling.enabled is set to true, mapreduce fails to launch with Invalid AMRMToken errors.
This error does not occur when the distributed scheduling option is disabled.
yarn-site.xml
<?xml version="1.0"?> <!-- Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. See accompanying LICENSE file. --> <configuration> <!-- Site specific YARN configuration properties --> <property> <name>yarn.resourcemanager.hostname</name> <value>h3master</value> </property> <property> <name>yarn.nodemanager.aux-services</name> <value>mapreduce_shuffle</value> </property> <property> <name>yarn.nodemanager.env-whitelist</name> <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value> </property> <property> <name>yarn.nodemanager.aux-services</name> <value>mapreduce_shuffle</value> </property> <property> <name>yarn.nodemanager.vmem-check-enabled</name> <value>false</value> </property> <property> <name>yarn.resourcemanager.opportunistic-container-allocation.enabled</name> <value>true</value> </property> <property> <name>yarn.nodemanager.opportunistic-containers-max-queue-length</name> <value>10</value> </property> <property> <name>yarn.nodemanager.distributed-scheduling.enabled</name> <value>true</value> </property> <property> <name>yarn.nodemanager.amrmproxy.enable</name> <value>true</value> </property> <property> <name>yarn.resourcemanager.opportunistic-container-allocation.enabled</name> <value>true</value> </property> <property> <name>yarn.nodemanager.resource.memory-mb</name> <value>4096</value> </property> </configuration>
Container Log
2017-04-23 05:17:50,324 INFO [main] org.apache.hadoop.mapreduce.v2.app.MRAppMaster: Created MRAppMaster for application appattempt_1492953411349_0001_000002 2017-04-23 05:17:51,625 INFO [main] org.apache.hadoop.mapreduce.v2.app.MRAppMaster: /************************************************************ [system properties] os.name: Linux os.version: 4.4.0-72-generic java.home: /usr/lib/jvm/java-8-openjdk-amd64/jre java.runtime.version: 1.8.0_121-8u121-b13-0ubuntu1.16.04.2-b13 java.vendor: Oracle Corporation java.version: 1.8.0_121 java.vm.name: OpenJDK 64-Bit Server VM java.class.path: /tmp/hadoop-administrator/nm-local-dir/usercache/administrator/appcache/application_1492953411349_0001/container_1492953411349_0001_02_000001:/home/administrator/hadoop-3.0.0-alpha2/etc/hadoop:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/hadoop-common-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/hadoop-common-3.0.0-alpha2-tests.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/hadoop-nfs-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/hadoop-kms-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jaxb-api-2.2.11.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/kerb-server-1.0.0-RC2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jetty-servlet-9.3.11.v20160721.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jetty-xml-9.3.11.v20160721.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jackson-jaxrs-1.9.13.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jsp-api-2.1.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/curator-client-2.7.1.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jline-0.9.94.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/curator-recipes-2.7.1.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/kerb-common-1.0.0-RC2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/xmlenc-0.52.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/kerb-simplekdc-1.0.0-RC2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/snappy-java-1.0.4.1.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jetty-webapp-9.3.11.v20160721.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jetty-http-9.3.11.v20160721.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/commons-configuration2-2.1.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/protobuf-java-2.5.0.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/commons-cli-1.2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/commons-compress-1.4.1.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/netty-3.10.5.Final.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/commons-net-3.1.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/commons-lang3-3.3.2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jcip-annotations-1.0.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/guava-11.0.2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/commons-io-2.4.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/httpcore-4.4.4.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jersey-server-1.19.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/nimbus-jose-jwt-3.9.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jetty-security-9.3.11.v20160721.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jetty-io-9.3.11.v20160721.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/mockito-all-1.8.5.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/kerby-config-1.0.0-RC2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jettison-1.1.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jersey-servlet-1.19.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jackson-databind-2.7.8.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/gson-2.2.4.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jetty-util-9.3.11.v20160721.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/commons-math3-3.1.1.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jaxb-impl-2.2.3-1.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jackson-mapper-asl-1.9.13.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jetty-server-9.3.11.v20160721.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/paranamer-2.3.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/kerby-util-1.0.0-RC2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/htrace-core4-4.1.0-incubating.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/commons-logging-1.1.3.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/re2j-1.0.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/kerb-identity-1.0.0-RC2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/hadoop-annotations-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/slf4j-log4j12-1.7.10.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/kerby-asn1-1.0.0-RC2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/kerb-util-1.0.0-RC2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/log4j-1.2.17.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jersey-json-1.19.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jackson-core-2.7.8.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/hamcrest-core-1.3.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/metrics-core-3.0.1.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jul-to-slf4j-1.7.10.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/xz-1.0.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jsr305-3.0.0.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/json-smart-1.1.1.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/commons-lang-2.6.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/httpclient-4.5.2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jsr311-api-1.1.1.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jets3t-0.9.0.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jsch-0.1.51.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/javax.servlet-api-3.1.0.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/commons-codec-1.4.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/java-xmlbuilder-0.4.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jackson-xc-1.9.13.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/commons-collections-3.2.2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/kerb-core-1.0.0-RC2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/commons-beanutils-1.9.3.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jersey-core-1.19.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jackson-core-asl-1.9.13.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/kerby-pkix-1.0.0-RC2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/jackson-annotations-2.7.8.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/zookeeper-3.4.6.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/kerb-crypto-1.0.0-RC2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/avro-1.7.4.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/kerb-client-1.0.0-RC2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/slf4j-api-1.7.10.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/junit-4.11.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/kerb-admin-1.0.0-RC2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/curator-framework-2.7.1.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/common/lib/hadoop-auth-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/hdfs/hadoop-hdfs-native-client-3.0.0-alpha2-tests.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/hdfs/hadoop-hdfs-3.0.0-alpha2-tests.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/hdfs/hadoop-hdfs-native-client-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/hdfs/hadoop-hdfs-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/hdfs/hadoop-hdfs-nfs-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/hdfs/hadoop-hdfs-client-3.0.0-alpha2-tests.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/hdfs/lib/xml-apis-1.3.04.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/hdfs/lib/hpack-0.11.0.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/hdfs/lib/xercesImpl-2.9.1.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/hdfs/lib/netty-all-4.1.0.Beta5.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/hdfs/lib/commons-daemon-1.0.13.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/hdfs/lib/hadoop-hdfs-client-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/hdfs/lib/okhttp-2.4.0.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/hdfs/lib/leveldbjni-all-1.8.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/hdfs/lib/jetty-util-ajax-9.3.11.v20160721.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/hdfs/lib/okio-1.4.0.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/hadoop-yarn-server-resourcemanager-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/hadoop-yarn-server-nodemanager-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/hadoop-yarn-server-tests-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/hadoop-yarn-applications-distributedshell-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/hadoop-yarn-registry-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/hadoop-yarn-server-timelineservice-hbase-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/hadoop-yarn-applications-unmanaged-am-launcher-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/hadoop-yarn-server-timelineservice-hbase-tests-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/hadoop-yarn-server-timelineservice-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/hadoop-yarn-server-sharedcachemanager-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/hadoop-yarn-api-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/hadoop-yarn-client-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/hadoop-yarn-server-web-proxy-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/hadoop-yarn-common-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/hadoop-yarn-server-common-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/hadoop-yarn-server-timeline-pluginstorage-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/hadoop-yarn-server-applicationhistoryservice-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/javax.inject-1.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/commons-math-2.2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/zookeeper-3.4.6-tests.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/findbugs-annotations-1.3.9-1.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/hbase-protocol-1.2.4.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/joni-2.1.2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/metrics-core-2.2.0.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/objenesis-2.1.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/hbase-hadoop-compat-1.2.4.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/guice-servlet-4.0.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/jamon-runtime-2.4.1.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/hbase-annotations-1.2.4.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/fst-2.24.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/hbase-client-1.2.4.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/jackson-jaxrs-json-provider-2.7.8.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/servlet-api-2.5-6.1.14.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/jackson-jaxrs-base-2.7.8.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/jasper-compiler-5.5.23.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/commons-el-1.0.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/jackson-module-jaxb-annotations-2.7.8.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/jcodings-1.0.8.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/jasper-runtime-5.5.23.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/guice-4.0.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/metrics-core-3.0.1.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/hbase-prefix-tree-1.2.4.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/aopalliance-1.0.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/commons-csv-1.0.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/javassist-3.18.1-GA.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/disruptor-3.3.0.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/commons-httpclient-3.1.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/hbase-hadoop2-compat-1.2.4.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/jsp-api-2.1-6.1.14.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/curator-test-2.7.1.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/hbase-common-1.2.4.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/jersey-client-1.19.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/jersey-guice-1.19.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/hbase-server-1.2.4.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/htrace-core-3.1.0-incubating.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/hbase-procedure-1.2.4.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/lib/jsp-2.1-6.1.14.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/mapreduce/hadoop-mapreduce-client-hs-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/mapreduce/hadoop-mapreduce-client-nativetask-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-3.0.0-alpha2-tests.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/mapreduce/hadoop-mapreduce-client-app-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/mapreduce/hadoop-mapreduce-client-shuffle-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/mapreduce/hadoop-mapreduce-client-common-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/mapreduce/hadoop-mapreduce-client-hs-plugins-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/mapreduce/hadoop-mapreduce-client-core-3.0.0-alpha2.jar:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/mapreduce/lib/*:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/tools/lib/hadoop-rumen-3.0.0-alpha2.jar:job.jar/job.jar:job.jar/classes/:job.jar/lib/*:/tmp/hadoop-administrator/nm-local-dir/usercache/administrator/appcache/application_1492953411349_0001/container_1492953411349_0001_02_000001/job.jar java.io.tmpdir: /tmp/hadoop-administrator/nm-local-dir/usercache/administrator/appcache/application_1492953411349_0001/container_1492953411349_0001_02_000001/tmp user.dir: /tmp/hadoop-administrator/nm-local-dir/usercache/administrator/appcache/application_1492953411349_0001/container_1492953411349_0001_02_000001 user.name: administrator ************************************************************/ 2017-04-23 05:17:51,801 INFO [main] org.apache.hadoop.security.SecurityUtil: Updating Configuration 2017-04-23 05:17:51,843 INFO [main] org.apache.hadoop.mapreduce.v2.app.MRAppMaster: Executing with tokens: 2017-04-23 05:17:51,844 INFO [main] org.apache.hadoop.mapreduce.v2.app.MRAppMaster: Kind: YARN_AM_RM_TOKEN, Service: , Ident: (appAttemptId { application_id { id: 1 cluster_timestamp: 1492953411349 } attemptId: 2 } keyId: -1667802072) 2017-04-23 05:17:52,086 INFO [main] org.apache.hadoop.mapreduce.v2.app.MRAppMaster: Using mapred newApiCommitter. 2017-04-23 05:17:52,089 INFO [main] org.apache.hadoop.mapreduce.v2.app.MRAppMaster: OutputCommitter set in config null 2017-04-23 05:17:52,190 INFO [main] org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter: File Output Committer Algorithm version is 2 2017-04-23 05:17:52,190 INFO [main] org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter: FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false 2017-04-23 05:17:53,745 INFO [main] org.apache.hadoop.mapreduce.v2.app.MRAppMaster: OutputCommitter is org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter 2017-04-23 05:17:54,054 INFO [main] org.apache.hadoop.yarn.event.AsyncDispatcher: Registering class org.apache.hadoop.mapreduce.jobhistory.EventType for class org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler 2017-04-23 05:17:54,056 INFO [main] org.apache.hadoop.yarn.event.AsyncDispatcher: Registering class org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType for class org.apache.hadoop.mapreduce.v2.app.MRAppMaster$JobEventDispatcher 2017-04-23 05:17:54,057 INFO [main] org.apache.hadoop.yarn.event.AsyncDispatcher: Registering class org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType for class org.apache.hadoop.mapreduce.v2.app.MRAppMaster$TaskEventDispatcher 2017-04-23 05:17:54,059 INFO [main] org.apache.hadoop.yarn.event.AsyncDispatcher: Registering class org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType for class org.apache.hadoop.mapreduce.v2.app.MRAppMaster$TaskAttemptEventDispatcher 2017-04-23 05:17:54,060 INFO [main] org.apache.hadoop.yarn.event.AsyncDispatcher: Registering class org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventType for class org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventHandler 2017-04-23 05:17:54,062 INFO [main] org.apache.hadoop.yarn.event.AsyncDispatcher: Registering class org.apache.hadoop.mapreduce.v2.app.speculate.Speculator$EventType for class org.apache.hadoop.mapreduce.v2.app.MRAppMaster$SpeculatorEventDispatcher 2017-04-23 05:17:54,063 INFO [main] org.apache.hadoop.yarn.event.AsyncDispatcher: Registering class org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator$EventType for class org.apache.hadoop.mapreduce.v2.app.MRAppMaster$ContainerAllocatorRouter 2017-04-23 05:17:54,064 INFO [main] org.apache.hadoop.yarn.event.AsyncDispatcher: Registering class org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher$EventType for class org.apache.hadoop.mapreduce.v2.app.MRAppMaster$ContainerLauncherRouter 2017-04-23 05:17:54,154 INFO [main] org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils: Default file system [hdfs://h3master:9000] 2017-04-23 05:17:54,206 INFO [main] org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils: Default file system [hdfs://h3master:9000] 2017-04-23 05:17:54,254 INFO [main] org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils: Default file system [hdfs://h3master:9000] 2017-04-23 05:17:54,269 INFO [main] org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler: Emitting job history data to the timeline server is not enabled 2017-04-23 05:17:54,274 INFO [main] org.apache.hadoop.mapreduce.v2.app.MRAppMaster: Attempting to recover. 2017-04-23 05:17:54,323 INFO [main] org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils: Default file system [hdfs://h3master:9000] 2017-04-23 05:17:54,327 INFO [main] org.apache.hadoop.mapreduce.v2.app.MRAppMaster: Previous history file is at hdfs://h3master:9000/tmp/hadoop-yarn/staging/administrator/.staging/job_1492953411349_0001/job_1492953411349_0001_1.jhist 2017-04-23 05:17:54,967 INFO [main] org.apache.hadoop.mapreduce.v2.app.MRAppMaster: Read completed tasks from history 0 2017-04-23 05:17:55,062 INFO [main] org.apache.hadoop.yarn.event.AsyncDispatcher: Registering class org.apache.hadoop.mapreduce.v2.app.job.event.JobFinishEvent$Type for class org.apache.hadoop.mapreduce.v2.app.MRAppMaster$JobFinishEventHandler 2017-04-23 05:17:55,255 INFO [main] org.apache.commons.beanutils.FluentPropertyBeanIntrospector: Error when creating PropertyDescriptor for public final void org.apache.commons.configuration2.AbstractConfiguration.setProperty(java.lang.String,java.lang.Object)! Ignoring this property. 2017-04-23 05:17:55,326 INFO [main] org.apache.hadoop.metrics2.impl.MetricsConfig: loaded properties from hadoop-metrics2.properties 2017-04-23 05:17:55,447 INFO [main] org.apache.hadoop.metrics2.impl.MetricsSystemImpl: Scheduled Metric snapshot period at 10 second(s). 2017-04-23 05:17:55,448 INFO [main] org.apache.hadoop.metrics2.impl.MetricsSystemImpl: MRAppMaster metrics system started 2017-04-23 05:17:55,465 INFO [main] org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl: Adding job token for job_1492953411349_0001 to jobTokenSecretManager 2017-04-23 05:17:55,500 INFO [main] org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl: Not uberizing job_1492953411349_0001 because: not enabled; too many maps; 2017-04-23 05:17:55,565 INFO [main] org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl: Input size for job job_1492953411349_0001 = 3540. Number of splits = 30 2017-04-23 05:17:55,568 INFO [main] org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl: Number of reduces for job job_1492953411349_0001 = 1 2017-04-23 05:17:55,568 INFO [main] org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl: job_1492953411349_0001Job Transitioned from NEW to INITED 2017-04-23 05:17:55,571 INFO [main] org.apache.hadoop.mapreduce.v2.app.MRAppMaster: MRAppMaster launching normal, non-uberized, multi-container job job_1492953411349_0001. 2017-04-23 05:17:55,627 INFO [main] org.apache.hadoop.ipc.CallQueueManager: Using callQueue: class java.util.concurrent.LinkedBlockingQueue queueCapacity: 100 scheduler: class org.apache.hadoop.ipc.DefaultRpcScheduler 2017-04-23 05:17:55,646 INFO [Socket Reader #1 for port 37204] org.apache.hadoop.ipc.Server: Starting Socket Reader #1 for port 37204 2017-04-23 05:17:55,711 INFO [main] org.apache.hadoop.yarn.factories.impl.pb.RpcServerFactoryPBImpl: Adding protocol org.apache.hadoop.mapreduce.v2.api.MRClientProtocolPB to the server 2017-04-23 05:17:55,712 INFO [IPC Server Responder] org.apache.hadoop.ipc.Server: IPC Server Responder: starting 2017-04-23 05:17:55,717 INFO [IPC Server listener on 37204] org.apache.hadoop.ipc.Server: IPC Server listener on 37204: starting 2017-04-23 05:17:55,720 INFO [main] org.apache.hadoop.mapreduce.v2.app.client.MRClientService: Instantiated MRClientService at se-vm-lz/192.168.0.180:37204 2017-04-23 05:17:55,788 INFO [main] org.eclipse.jetty.util.log: Logging initialized @6158ms 2017-04-23 05:17:55,977 INFO [main] org.apache.hadoop.security.authentication.server.AuthenticationFilter: Unable to initialize FileSignerSecretProvider, falling back to use random secrets. 2017-04-23 05:17:55,982 INFO [main] org.apache.hadoop.http.HttpRequestLog: Http request log for http.requests.mapreduce is not defined 2017-04-23 05:17:55,169 INFO [main] org.apache.hadoop.http.HttpServer2: Added global filter 'safety' (class=org.apache.hadoop.http.HttpServer2$QuotingInputFilter) 2017-04-23 05:17:55,187 INFO [main] org.apache.hadoop.http.HttpServer2: Added filter AM_PROXY_FILTER (class=org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter) to context mapreduce 2017-04-23 05:17:55,187 INFO [main] org.apache.hadoop.http.HttpServer2: Added filter AM_PROXY_FILTER (class=org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter) to context static 2017-04-23 05:17:55,193 INFO [main] org.apache.hadoop.http.HttpServer2: adding path spec: /mapreduce/* 2017-04-23 05:17:55,193 INFO [main] org.apache.hadoop.http.HttpServer2: adding path spec: /ws/* 2017-04-23 05:17:57,018 INFO [main] org.apache.hadoop.yarn.webapp.WebApps: Registered webapp guice modules 2017-04-23 05:17:57,021 INFO [main] org.apache.hadoop.http.HttpServer2: Jetty bound to port 41728 2017-04-23 05:17:57,023 INFO [main] org.eclipse.jetty.server.Server: jetty-9.3.11.v20160721 2017-04-23 05:17:57,098 INFO [main] org.eclipse.jetty.server.handler.ContextHandler: Started o.e.j.s.ServletContextHandler@f4c0e4e{/static,jar:file:/home/administrator/hadoop-3.0.0-alpha2/share/hadoop/yarn/hadoop-yarn-common-3.0.0-alpha2.jar!/webapps/static,AVAILABLE} 2017-04-23 05:17:58,887 INFO [main] org.eclipse.jetty.server.handler.ContextHandler: Started o.e.j.w.WebAppContext@6c184d4d{/,file:///tmp/hadoop-administrator/nm-local-dir/usercache/administrator/appcache/application_1492953411349_0001/container_1492953411349_0001_02_000001/tmp/jetty-0.0.0.0-41728-mapreduce-_-any-5639695627459918337.dir/webapp/,AVAILABLE}{/mapreduce} 2017-04-23 05:17:58,896 INFO [main] org.eclipse.jetty.server.AbstractConnector: Started ServerConnector@5f34907b{HTTP/1.1,[http/1.1]}{0.0.0.0:41728} 2017-04-23 05:17:58,897 INFO [main] org.eclipse.jetty.server.Server: Started @9273ms 2017-04-23 05:17:58,898 INFO [main] org.apache.hadoop.yarn.webapp.WebApps: Web app mapreduce started at 41728 2017-04-23 05:17:58,906 INFO [main] org.apache.hadoop.ipc.CallQueueManager: Using callQueue: class java.util.concurrent.LinkedBlockingQueue queueCapacity: 3000 scheduler: class org.apache.hadoop.ipc.DefaultRpcScheduler 2017-04-23 05:17:58,907 INFO [Socket Reader #1 for port 35802] org.apache.hadoop.ipc.Server: Starting Socket Reader #1 for port 35802 2017-04-23 05:17:58,915 INFO [IPC Server Responder] org.apache.hadoop.ipc.Server: IPC Server Responder: starting 2017-04-23 05:17:58,915 INFO [IPC Server listener on 35802] org.apache.hadoop.ipc.Server: IPC Server listener on 35802: starting 2017-04-23 05:17:58,983 INFO [main] org.apache.hadoop.mapreduce.v2.app.rm.RMContainerRequestor: nodeBlacklistingEnabled:true 2017-04-23 05:17:58,983 INFO [main] org.apache.hadoop.mapreduce.v2.app.rm.RMContainerRequestor: maxTaskFailuresPerNode is 3 2017-04-23 05:17:58,983 INFO [main] org.apache.hadoop.mapreduce.v2.app.rm.RMContainerRequestor: blacklistDisablePercent is 33 2017-04-23 05:17:58,989 INFO [main] org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator: 0% of the mappers will be scheduled using OPPORTUNISTIC containers 2017-04-23 05:17:59,071 INFO [main] org.apache.hadoop.yarn.client.RMProxy: Connecting to ResourceManager at h3master/192.168.0.165:8030 2017-04-23 05:17:59,170 WARN [main] org.apache.hadoop.ipc.Client: Exception encountered while connecting to the server : org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken): Invalid AMRMToken from appattempt_1492953411349_0001_000002 2017-04-23 05:17:59,178 ERROR [main] org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator: Exception while registering org.apache.hadoop.security.token.SecretManager$InvalidToken: Invalid AMRMToken from appattempt_1492953411349_0001_000002 at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateException(RPCUtil.java:53) at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateIOException(RPCUtil.java:80) at org.apache.hadoop.yarn.ipc.RPCUtil.unwrapAndThrowException(RPCUtil.java:119) at org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.registerApplicationMaster(ApplicationMasterProtocolPBClientImpl.java:109) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:398) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:163) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:155) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:335) at com.sun.proxy.$Proxy86.registerApplicationMaster(Unknown Source) at org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.register(RMCommunicator.java:161) at org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.serviceStart(RMCommunicator.java:121) at org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator.serviceStart(RMContainerAllocator.java:280) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:193) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster$ContainerAllocatorRouter.serviceStart(MRAppMaster.java:966) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:193) at org.apache.hadoop.service.CompositeService.serviceStart(CompositeService.java:120) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.serviceStart(MRAppMaster.java:1263) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:193) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster$6.run(MRAppMaster.java:1716) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1857) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.initAndStartAppMaster(MRAppMaster.java:1712) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.main(MRAppMaster.java:1643) Caused by: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken): Invalid AMRMToken from appattempt_1492953411349_0001_000002 at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1481) at org.apache.hadoop.ipc.Client.call(Client.java:1427) at org.apache.hadoop.ipc.Client.call(Client.java:1337) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:227) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:115) at com.sun.proxy.$Proxy85.registerApplicationMaster(Unknown Source) at org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.registerApplicationMaster(ApplicationMasterProtocolPBClientImpl.java:107) ... 25 more 2017-04-23 05:17:59,183 INFO [main] org.apache.hadoop.service.AbstractService: Service RMCommunicator failed in state STARTED; cause: org.apache.hadoop.yarn.exceptions.YarnRuntimeException: org.apache.hadoop.security.token.SecretManager$InvalidToken: Invalid AMRMToken from appattempt_1492953411349_0001_000002 org.apache.hadoop.yarn.exceptions.YarnRuntimeException: org.apache.hadoop.security.token.SecretManager$InvalidToken: Invalid AMRMToken from appattempt_1492953411349_0001_000002 at org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.register(RMCommunicator.java:177) at org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.serviceStart(RMCommunicator.java:121) at org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator.serviceStart(RMContainerAllocator.java:280) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:193) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster$ContainerAllocatorRouter.serviceStart(MRAppMaster.java:966) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:193) at org.apache.hadoop.service.CompositeService.serviceStart(CompositeService.java:120) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.serviceStart(MRAppMaster.java:1263) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:193) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster$6.run(MRAppMaster.java:1716) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1857) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.initAndStartAppMaster(MRAppMaster.java:1712) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.main(MRAppMaster.java:1643) Caused by: org.apache.hadoop.security.token.SecretManager$InvalidToken: Invalid AMRMToken from appattempt_1492953411349_0001_000002 at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateException(RPCUtil.java:53) at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateIOException(RPCUtil.java:80) at org.apache.hadoop.yarn.ipc.RPCUtil.unwrapAndThrowException(RPCUtil.java:119) at org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.registerApplicationMaster(ApplicationMasterProtocolPBClientImpl.java:109) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:398) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:163) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:155) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:335) at com.sun.proxy.$Proxy86.registerApplicationMaster(Unknown Source) at org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.register(RMCommunicator.java:161) ... 14 more Caused by: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken): Invalid AMRMToken from appattempt_1492953411349_0001_000002 at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1481) at org.apache.hadoop.ipc.Client.call(Client.java:1427) at org.apache.hadoop.ipc.Client.call(Client.java:1337) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:227) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:115) at com.sun.proxy.$Proxy85.registerApplicationMaster(Unknown Source) at org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.registerApplicationMaster(ApplicationMasterProtocolPBClientImpl.java:107) ... 25 more 2017-04-23 05:17:59,186 INFO [main] org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator: Final Stats: PendingReds:0 ScheduledMaps:0 ScheduledReds:0 AssignedMaps:0 AssignedReds:0 CompletedMaps:0 CompletedReds:0 ContAlloc:0 ContRel:0 HostLocal:0 RackLocal:0 2017-04-23 05:17:59,186 INFO [main] org.apache.hadoop.service.AbstractService: Service org.apache.hadoop.mapreduce.v2.app.MRAppMaster$ContainerAllocatorRouter failed in state STARTED; cause: org.apache.hadoop.yarn.exceptions.YarnRuntimeException: org.apache.hadoop.security.token.SecretManager$InvalidToken: Invalid AMRMToken from appattempt_1492953411349_0001_000002 org.apache.hadoop.yarn.exceptions.YarnRuntimeException: org.apache.hadoop.security.token.SecretManager$InvalidToken: Invalid AMRMToken from appattempt_1492953411349_0001_000002 at org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.register(RMCommunicator.java:177) at org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.serviceStart(RMCommunicator.java:121) at org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator.serviceStart(RMContainerAllocator.java:280) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:193) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster$ContainerAllocatorRouter.serviceStart(MRAppMaster.java:966) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:193) at org.apache.hadoop.service.CompositeService.serviceStart(CompositeService.java:120) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.serviceStart(MRAppMaster.java:1263) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:193) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster$6.run(MRAppMaster.java:1716) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1857) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.initAndStartAppMaster(MRAppMaster.java:1712) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.main(MRAppMaster.java:1643) Caused by: org.apache.hadoop.security.token.SecretManager$InvalidToken: Invalid AMRMToken from appattempt_1492953411349_0001_000002 at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateException(RPCUtil.java:53) at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateIOException(RPCUtil.java:80) at org.apache.hadoop.yarn.ipc.RPCUtil.unwrapAndThrowException(RPCUtil.java:119) at org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.registerApplicationMaster(ApplicationMasterProtocolPBClientImpl.java:109) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:398) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:163) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:155) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:335) at com.sun.proxy.$Proxy86.registerApplicationMaster(Unknown Source) at org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.register(RMCommunicator.java:161) ... 14 more Caused by: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken): Invalid AMRMToken from appattempt_1492953411349_0001_000002 at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1481) at org.apache.hadoop.ipc.Client.call(Client.java:1427) at org.apache.hadoop.ipc.Client.call(Client.java:1337) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:227) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:115) at com.sun.proxy.$Proxy85.registerApplicationMaster(Unknown Source) at org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.registerApplicationMaster(ApplicationMasterProtocolPBClientImpl.java:107) ... 25 more 2017-04-23 05:17:59,188 INFO [main] org.apache.hadoop.service.AbstractService: Service org.apache.hadoop.mapreduce.v2.app.MRAppMaster failed in state STARTED; cause: org.apache.hadoop.yarn.exceptions.YarnRuntimeException: org.apache.hadoop.security.token.SecretManager$InvalidToken: Invalid AMRMToken from appattempt_1492953411349_0001_000002 org.apache.hadoop.yarn.exceptions.YarnRuntimeException: org.apache.hadoop.security.token.SecretManager$InvalidToken: Invalid AMRMToken from appattempt_1492953411349_0001_000002 at org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.register(RMCommunicator.java:177) at org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.serviceStart(RMCommunicator.java:121) at org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator.serviceStart(RMContainerAllocator.java:280) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:193) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster$ContainerAllocatorRouter.serviceStart(MRAppMaster.java:966) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:193) at org.apache.hadoop.service.CompositeService.serviceStart(CompositeService.java:120) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.serviceStart(MRAppMaster.java:1263) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:193) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster$6.run(MRAppMaster.java:1716) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1857) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.initAndStartAppMaster(MRAppMaster.java:1712) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.main(MRAppMaster.java:1643) Caused by: org.apache.hadoop.security.token.SecretManager$InvalidToken: Invalid AMRMToken from appattempt_1492953411349_0001_000002 at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateException(RPCUtil.java:53) at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateIOException(RPCUtil.java:80) at org.apache.hadoop.yarn.ipc.RPCUtil.unwrapAndThrowException(RPCUtil.java:119) at org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.registerApplicationMaster(ApplicationMasterProtocolPBClientImpl.java:109) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:398) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:163) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:155) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:335) at com.sun.proxy.$Proxy86.registerApplicationMaster(Unknown Source) at org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.register(RMCommunicator.java:161) ... 14 more Caused by: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken): Invalid AMRMToken from appattempt_1492953411349_0001_000002 at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1481) at org.apache.hadoop.ipc.Client.call(Client.java:1427) at org.apache.hadoop.ipc.Client.call(Client.java:1337) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:227) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:115) at com.sun.proxy.$Proxy85.registerApplicationMaster(Unknown Source) at org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.registerApplicationMaster(ApplicationMasterProtocolPBClientImpl.java:107) ... 25 more 2017-04-23 05:17:59,191 INFO [main] org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler: Stopping JobHistoryEventHandler. Size of the outstanding queue size is 3 2017-04-23 05:17:59,192 INFO [main] org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler: In stop, writing event AM_STARTED 2017-04-23 05:17:59,312 INFO [main] org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler: Event Writer setup for JobId: job_1492953411349_0001, File: hdfs://h3master:9000/tmp/hadoop-yarn/staging/administrator/.staging/job_1492953411349_0001/job_1492953411349_0001_2.jhist 2017-04-23 05:17:59,640 INFO [main] org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler: In stop, writing event AM_STARTED 2017-04-23 05:17:59,642 INFO [main] org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler: In stop, writing event JOB_SUBMITTED 2017-04-23 05:17:59,694 INFO [main] org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler: Stopped JobHistoryEventHandler. super.stop() 2017-04-23 05:17:59,695 INFO [main] org.apache.hadoop.mapreduce.v2.app.MRAppMaster: Skipping cleaning up the staging dir. assuming AM will be retried. 2017-04-23 05:17:59,695 INFO [main] org.apache.hadoop.ipc.Server: Stopping server on 35802 2017-04-23 05:17:59,698 INFO [IPC Server listener on 35802] org.apache.hadoop.ipc.Server: Stopping IPC Server listener on 35802 2017-04-23 05:17:59,698 INFO [IPC Server Responder] org.apache.hadoop.ipc.Server: Stopping IPC Server Responder 2017-04-23 05:17:59,699 INFO [Ping Checker] org.apache.hadoop.yarn.util.AbstractLivelinessMonitor: TaskAttemptFinishingMonitor thread interrupted 2017-04-23 05:17:59,699 INFO [TaskHeartbeatHandler PingChecker] org.apache.hadoop.mapreduce.v2.app.TaskHeartbeatHandler: TaskHeartbeatHandler thread interrupted 2017-04-23 05:17:59,703 FATAL [main] org.apache.hadoop.mapreduce.v2.app.MRAppMaster: Error starting MRAppMaster org.apache.hadoop.yarn.exceptions.YarnRuntimeException: org.apache.hadoop.security.token.SecretManager$InvalidToken: Invalid AMRMToken from appattempt_1492953411349_0001_000002 at org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.register(RMCommunicator.java:177) at org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.serviceStart(RMCommunicator.java:121) at org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator.serviceStart(RMContainerAllocator.java:280) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:193) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster$ContainerAllocatorRouter.serviceStart(MRAppMaster.java:966) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:193) at org.apache.hadoop.service.CompositeService.serviceStart(CompositeService.java:120) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.serviceStart(MRAppMaster.java:1263) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:193) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster$6.run(MRAppMaster.java:1716) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1857) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.initAndStartAppMaster(MRAppMaster.java:1712) at org.apache.hadoop.mapreduce.v2.app.MRAppMaster.main(MRAppMaster.java:1643) Caused by: org.apache.hadoop.security.token.SecretManager$InvalidToken: Invalid AMRMToken from appattempt_1492953411349_0001_000002 at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateException(RPCUtil.java:53) at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateIOException(RPCUtil.java:80) at org.apache.hadoop.yarn.ipc.RPCUtil.unwrapAndThrowException(RPCUtil.java:119) at org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.registerApplicationMaster(ApplicationMasterProtocolPBClientImpl.java:109) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:398) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:163) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:155) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:335) at com.sun.proxy.$Proxy86.registerApplicationMaster(Unknown Source) at org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.register(RMCommunicator.java:161) ... 14 more Caused by: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken): Invalid AMRMToken from appattempt_1492953411349_0001_000002 at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1481) at org.apache.hadoop.ipc.Client.call(Client.java:1427) at org.apache.hadoop.ipc.Client.call(Client.java:1337) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:227) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:115) at com.sun.proxy.$Proxy85.registerApplicationMaster(Unknown Source) at org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.registerApplicationMaster(ApplicationMasterProtocolPBClientImpl.java:107) ... 25 more 2017-04-23 05:17:59,709 INFO [main] org.apache.hadoop.util.ExitUtil: Exiting with status 1