[centos@ip-172-31-69-167 ~]$ sudo -u hive yarn logs -applicationId application_1555415227039_0004 19/04/16 12:42:23 INFO client.RMProxy: Connecting to ResourceManager at ip-172-31-69-167.ec2.internal/172.31.69.167:8050 19/04/16 12:42:23 INFO client.AHSProxy: Connecting to Application History server at ip-172-31-69-167.ec2.internal/172.31.69.167:10200 19/04/16 12:42:24 INFO zlib.ZlibFactory: Successfully loaded & initialized native-zlib library 19/04/16 12:42:24 INFO compress.CodecPool: Got brand-new decompressor [.deflate] Container: container_e06_1555415227039_0004_01_000001 on ip-172-31-83-216.ec2.internal_45454_1555418368508 LogAggregationType: AGGREGATED ========================================================================================================== LogType:dag_1555415227039_0004_1.dot LogLastModifiedTime:Tue Apr 16 12:39:28 +0000 2019 LogLength:1031 LogContents: digraph Select_T_Candidate_startTime_T_Candidat_T_Stage_1_ { graph [ label="Select_T_Candidate_startTime_T_Candidat_T_Stage_1_", fontsize=24, fontname=Helvetica]; node [fontsize=12, fontname=Helvetica]; edge [fontsize=9, fontcolor=blue, fontname=Arial]; "Select_T_Candidate_startTime_T_Candidat_T_Stage_1_.Map_1" [ label = "Map_1[MapTezProcessor]" , color= "black" ]; "Select_T_Candidate_startTime_T_Candidat_T_Stage_1_.Map_1" -> "Select_T_Candidate_startTime_T_Candidat_T_Stage_1_.Map_1_out_Map_1" [ label = "Output [outputClass=MROutput,\n committer=]" ]; "Select_T_Candidate_startTime_T_Candidat_T_Stage_1_.Map_1_idtable" [ label = "Map_1[idtable]", shape = "box" , color= "black"]; "Select_T_Candidate_startTime_T_Candidat_T_Stage_1_.Map_1_idtable" -> "Select_T_Candidate_startTime_T_Candidat_T_Stage_1_.Map_1" [ label = "Input [inputClass=MRInputLegacy,\n initializer=HiveSplitGenerator]" ]; "Select_T_Candidate_startTime_T_Candidat_T_Stage_1_.Map_1_out_Map_1" [ label = "Map_1[out_Map_1]", shape = "box" , color= "black"]; } End of LogType:dag_1555415227039_0004_1.dot ********************************************************************************************* Container: container_e06_1555415227039_0004_01_000001 on ip-172-31-83-216.ec2.internal_45454_1555418368508 LogAggregationType: AGGREGATED ========================================================================================================== LogType:directory.info LogLastModifiedTime:Tue Apr 16 12:39:28 +0000 2019 LogLength:23395 LogContents: ls -l: total 20 -rw-r--r-- 1 yarn hadoop 174 Apr 16 12:37 container_tokens -rwx------ 1 yarn hadoop 652 Apr 16 12:37 default_container_executor_session.sh -rwx------ 1 yarn hadoop 707 Apr 16 12:37 default_container_executor.sh lrwxrwxrwx 1 yarn hadoop 143 Apr 16 12:37 hive-exec-3.1.0.3.0.1.0-187-ffcc945cef1d45a4f8c86dab4e04eaea5bb4284dae0788c8b7f4df27141566d9.jar -> /hadoop/yarn/local/usercache/hive/filecache/10/hive-exec-3.1.0.3.0.1.0-187-ffcc945cef1d45a4f8c86dab4e04eaea5bb4284dae0788c8b7f4df27141566d9.jar lrwxrwxrwx 1 yarn hadoop 69 Apr 16 12:37 hive-hcatalog-core.jar -> /hadoop/yarn/local/usercache/hive/filecache/16/hive-hcatalog-core.jar -rwx------ 1 yarn hadoop 5409 Apr 16 12:37 launch_container.sh lrwxrwxrwx 1 yarn hadoop 98 Apr 16 12:37 tez-conf.pb -> /hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/filecache/11/tez-conf.pb lrwxrwxrwx 1 yarn hadoop 42 Apr 16 12:37 tezlib -> /hadoop/yarn/local/filecache/10/tez.tar.gz lrwxrwxrwx 1 yarn hadoop 117 Apr 16 12:37 tez.session.local-resources.pb -> /hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/filecache/10/tez.session.local-resources.pb lrwxrwxrwx 1 yarn hadoop 76 Apr 16 12:37 TimestampUDF-1.0-SNAPSHOT.jar -> /hadoop/yarn/local/usercache/hive/filecache/17/TimestampUDF-1.0-SNAPSHOT.jar drwx--x--- 2 yarn hadoop 6 Apr 16 12:37 tmp find -L . -maxdepth 5 -ls: 436223266 4 drwx--x--- 3 yarn hadoop 4096 Apr 16 12:37 . 444607906 0 drwx--x--- 2 yarn hadoop 6 Apr 16 12:37 ./tmp 436223267 4 -rw-r--r-- 1 yarn hadoop 174 Apr 16 12:37 ./container_tokens 436223268 4 -rw-r--r-- 1 yarn hadoop 12 Apr 16 12:37 ./.container_tokens.crc 436223269 8 -rwx------ 1 yarn hadoop 5409 Apr 16 12:37 ./launch_container.sh 436223272 4 -rw-r--r-- 1 yarn hadoop 52 Apr 16 12:37 ./.launch_container.sh.crc 436223273 4 -rwx------ 1 yarn hadoop 652 Apr 16 12:37 ./default_container_executor_session.sh 436223274 4 -rw-r--r-- 1 yarn hadoop 16 Apr 16 12:37 ./.default_container_executor_session.sh.crc 436223283 4 -rwx------ 1 yarn hadoop 707 Apr 16 12:37 ./default_container_executor.sh 436223284 4 -rw-r--r-- 1 yarn hadoop 16 Apr 16 12:37 ./.default_container_executor.sh.crc 260050168 40332 -r-x------ 1 yarn hadoop 41297919 Apr 8 09:22 ./hive-exec-3.1.0.3.0.1.0-187-ffcc945cef1d45a4f8c86dab4e04eaea5bb4284dae0788c8b7f4df27141566d9.jar 415236418 4 -r-x------ 1 yarn hadoop 963 Apr 16 12:37 ./tez.session.local-resources.pb 406894318 4 -r-x------ 1 yarn hadoop 3909 Apr 16 12:37 ./TimestampUDF-1.0-SNAPSHOT.jar 423643714 124 -r-x------ 1 yarn hadoop 125998 Apr 16 12:37 ./tez-conf.pb 251662280 4 drwxr-xr-x 3 yarn hadoop 4096 Apr 8 09:06 ./tezlib 251662281 4 -r-xr-xr-x 1 yarn hadoop 1465 Sep 19 2018 ./tezlib/LICENSE-BSD-3clause 251662282 20 -r-xr-xr-x 1 yarn hadoop 17257 Sep 19 2018 ./tezlib/LICENSE 251666080 4 -r-xr-xr-x 1 yarn hadoop 1509 Sep 19 2018 ./tezlib/NOTICE 251666081 20 -r-xr-xr-x 1 yarn hadoop 16792 Sep 19 2018 ./tezlib/LICENSE-CDDLv1.0 251666082 4 -r-xr-xr-x 1 yarn hadoop 1045 Sep 19 2018 ./tezlib/LICENSE-MIT 251666083 8 -r-xr-xr-x 1 yarn hadoop 4128 Sep 19 2018 ./tezlib/LICENSE-SIL_OpenFontLicense-v1.1 251666084 36 -r-xr-xr-x 1 yarn hadoop 36261 Sep 19 2018 ./tezlib/LICENSE-CDDLv1.1-GPLv2_withCPE 251666085 12 -r-xr-xr-x 1 yarn hadoop 8899 Sep 19 2018 ./tezlib/hadoop-shim-0.9.1.3.0.1.0-187.jar 251666086 1020 -r-xr-xr-x 1 yarn hadoop 1041820 Sep 19 2018 ./tezlib/tez-api-0.9.1.3.0.1.0-187.jar 251666087 84 -r-xr-xr-x 1 yarn hadoop 85946 Sep 19 2018 ./tezlib/tez-common-0.9.1.3.0.1.0-187.jar 251666088 196 -r-xr-xr-x 1 yarn hadoop 199548 Sep 19 2018 ./tezlib/tez-runtime-internals-0.9.1.3.0.1.0-187.jar 251666089 752 -r-xr-xr-x 1 yarn hadoop 768635 Sep 19 2018 ./tezlib/tez-runtime-library-0.9.1.3.0.1.0-187.jar 251666090 292 -r-xr-xr-x 1 yarn hadoop 296385 Sep 19 2018 ./tezlib/tez-mapreduce-0.9.1.3.0.1.0-187.jar 251666091 56 -r-xr-xr-x 1 yarn hadoop 56876 Sep 19 2018 ./tezlib/tez-examples-0.9.1.3.0.1.0-187.jar 251666092 1380 -r-xr-xr-x 1 yarn hadoop 1409888 Sep 19 2018 ./tezlib/tez-dag-0.9.1.3.0.1.0-187.jar 251666093 156 -r-xr-xr-x 1 yarn hadoop 158850 Sep 19 2018 ./tezlib/tez-tests-0.9.1.3.0.1.0-187.jar 251666094 108 -r-xr-xr-x 1 yarn hadoop 107597 Sep 19 2018 ./tezlib/tez-ext-service-tests-0.9.1.3.0.1.0-187.jar 251666095 108616 -r-xr-xr-x 1 yarn hadoop 111220116 Sep 19 2018 ./tezlib/tez-ui-0.9.1.3.0.1.0-187.war 251666096 76 -r-xr-xr-x 1 yarn hadoop 75760 Sep 19 2018 ./tezlib/tez-protobuf-history-plugin-0.9.1.3.0.1.0-187.jar 251666097 28 -r-xr-xr-x 1 yarn hadoop 28273 Sep 19 2018 ./tezlib/tez-yarn-timeline-history-0.9.1.3.0.1.0-187.jar 251666098 8 -r-xr-xr-x 1 yarn hadoop 7839 Sep 19 2018 ./tezlib/tez-yarn-timeline-history-with-acls-0.9.1.3.0.1.0-187.jar 251666099 12 -r-xr-xr-x 1 yarn hadoop 11563 Sep 19 2018 ./tezlib/tez-yarn-timeline-cache-plugin-0.9.1.3.0.1.0-187.jar 251666100 24 -r-xr-xr-x 1 yarn hadoop 23691 Sep 19 2018 ./tezlib/tez-yarn-timeline-history-with-fs-0.9.1.3.0.1.0-187.jar 251666101 80 -r-xr-xr-x 1 yarn hadoop 79219 Sep 19 2018 ./tezlib/tez-history-parser-0.9.1.3.0.1.0-187.jar 251666102 80 -r-xr-xr-x 1 yarn hadoop 79101 Sep 19 2018 ./tezlib/tez-job-analyzer-0.9.1.3.0.1.0-187.jar 251666103 16 -r-xr-xr-x 1 yarn hadoop 15352 Sep 19 2018 ./tezlib/tez-javadoc-tools-0.9.1.3.0.1.0-187.jar 251666104 8 -r-xr-xr-x 1 yarn hadoop 6445 Sep 19 2018 ./tezlib/hadoop-shim-2.8-0.9.1.3.0.1.0-187.jar 297831968 12 drwxr-xr-x 3 yarn hadoop 8192 Apr 8 09:06 ./tezlib/lib 297831969 32 -r-xr-xr-x 1 yarn hadoop 32119 Jun 22 2018 ./tezlib/lib/slf4j-api-1.7.10.jar 297831970 3028 -r-xr-xr-x 1 yarn hadoop 3098999 Sep 19 2018 ./tezlib/lib/hadoop-yarn-api-3.1.1.3.0.1.0-187.jar 297831971 280 -r-xr-xr-x 1 yarn hadoop 284220 Jun 22 2018 ./tezlib/lib/commons-lang-2.6.jar 297831972 1612 -r-xr-xr-x 1 yarn hadoop 1648200 Sep 19 2018 ./tezlib/lib/guava-11.0.2.jar 297831973 36 -r-xr-xr-x 1 yarn hadoop 33031 Sep 19 2018 ./tezlib/lib/jsr305-3.0.0.jar 297831981 64 -r-xr-xr-x 1 yarn hadoop 62050 Jun 22 2018 ./tezlib/lib/commons-logging-1.1.3.jar 297831982 100 -r-xr-xr-x 1 yarn hadoop 102244 Jun 22 2018 ./tezlib/lib/jaxb-api-2.2.11.jar 297831983 524 -r-xr-xr-x 1 yarn hadoop 533455 Sep 19 2018 ./tezlib/lib/protobuf-java-2.5.0.jar 297831984 68 -r-xr-xr-x 1 yarn hadoop 66983 Jun 22 2018 ./tezlib/lib/jackson-annotations-2.9.5.jar 297831985 3952 -r-xr-xr-x 1 yarn hadoop 4043890 Sep 19 2018 ./tezlib/lib/hadoop-common-3.1.1.3.0.1.0-187.jar 297831986 60 -r-xr-xr-x 1 yarn hadoop 59895 Sep 19 2018 ./tezlib/lib/hadoop-annotations-3.1.1.3.0.1.0-187.jar 297831987 44 -r-xr-xr-x 1 yarn hadoop 41123 Jun 22 2018 ./tezlib/lib/commons-cli-1.2.jar 297831988 1564 -r-xr-xr-x 1 yarn hadoop 1599627 Jun 22 2018 ./tezlib/lib/commons-math3-3.1.1.jar 297831989 720 -r-xr-xr-x 1 yarn hadoop 736658 Jun 22 2018 ./tezlib/lib/httpclient-4.5.2.jar 297831990 320 -r-xr-xr-x 1 yarn hadoop 326724 Jun 22 2018 ./tezlib/lib/httpcore-4.4.4.jar 297831991 60 -r-xr-xr-x 1 yarn hadoop 58160 Jun 22 2018 ./tezlib/lib/commons-codec-1.4.jar 297831992 184 -r-xr-xr-x 1 yarn hadoop 185140 Jun 22 2018 ./tezlib/lib/commons-io-2.4.jar 297831993 304 -r-xr-xr-x 1 yarn hadoop 307410 Jun 22 2018 ./tezlib/lib/commons-net-3.6.jar 297831994 576 -r-xr-xr-x 1 yarn hadoop 588337 Jun 22 2018 ./tezlib/lib/commons-collections-3.2.2.jar 297831995 96 -r-xr-xr-x 1 yarn hadoop 95806 Jun 22 2018 ./tezlib/lib/javax.servlet-api-3.1.0.jar 297831996 120 -r-xr-xr-x 1 yarn hadoop 119119 Jun 22 2018 ./tezlib/lib/jetty-servlet-9.3.19.v20170502.jar 297831997 96 -r-xr-xr-x 1 yarn hadoop 95838 Jun 22 2018 ./tezlib/lib/jetty-security-9.3.19.v20170502.jar 297831998 112 -r-xr-xr-x 1 yarn hadoop 113395 Jun 22 2018 ./tezlib/lib/jetty-webapp-9.3.19.v20170502.jar 297831999 52 -r-xr-xr-x 1 yarn hadoop 50633 Jun 22 2018 ./tezlib/lib/jetty-xml-9.3.19.v20170502.jar 297832000 128 -r-xr-xr-x 1 yarn hadoop 128719 Jun 22 2018 ./tezlib/lib/jersey-servlet-1.19.jar 297832001 164 -r-xr-xr-x 1 yarn hadoop 165345 Jun 22 2018 ./tezlib/lib/jersey-json-1.19.jar 297832002 80 -r-xr-xr-x 1 yarn hadoop 81743 Jun 22 2018 ./tezlib/lib/jettison-1.3.4.jar 297832003 872 -r-xr-xr-x 1 yarn hadoop 890168 Jun 22 2018 ./tezlib/lib/jaxb-impl-2.2.3-1.jar 297832004 224 -r-xr-xr-x 1 yarn hadoop 228286 Jun 22 2018 ./tezlib/lib/jackson-core-asl-1.9.2.jar 297832005 748 -r-xr-xr-x 1 yarn hadoop 765648 Jun 22 2018 ./tezlib/lib/jackson-mapper-asl-1.9.2.jar 297832006 20 -r-xr-xr-x 1 yarn hadoop 18323 Jun 22 2018 ./tezlib/lib/jackson-jaxrs-1.9.2.jar 297832007 28 -r-xr-xr-x 1 yarn hadoop 27075 Jun 22 2018 ./tezlib/lib/jackson-xc-1.9.2.jar 297832008 480 -r-xr-xr-x 1 yarn hadoop 489884 Jun 22 2018 ./tezlib/lib/log4j-1.2.17.jar 297832009 244 -r-xr-xr-x 1 yarn hadoop 246174 Jun 22 2018 ./tezlib/lib/commons-beanutils-1.9.3.jar 297832010 604 -r-xr-xr-x 1 yarn hadoop 616888 Jun 22 2018 ./tezlib/lib/commons-configuration2-2.1.1.jar 297832011 428 -r-xr-xr-x 1 yarn hadoop 434678 Jun 22 2018 ./tezlib/lib/commons-lang3-3.4.jar 297832012 12 -r-xr-xr-x 1 yarn hadoop 8866 Jun 22 2018 ./tezlib/lib/slf4j-log4j12-1.7.10.jar 297832013 428 -r-xr-xr-x 1 yarn hadoop 436303 Jun 22 2018 ./tezlib/lib/avro-1.7.7.jar 297832014 32 -r-xr-xr-x 1 yarn hadoop 29555 Jun 22 2018 ./tezlib/lib/paranamer-2.3.jar 297832015 1224 -r-xr-xr-x 1 yarn hadoop 1251514 Jun 22 2018 ./tezlib/lib/snappy-java-1.0.5.jar 297832016 236 -r-xr-xr-x 1 yarn hadoop 241367 Jun 22 2018 ./tezlib/lib/commons-compress-1.4.1.jar 297832017 96 -r-xr-xr-x 1 yarn hadoop 94672 Jun 22 2018 ./tezlib/lib/xz-1.0.jar 297832018 128 -r-xr-xr-x 1 yarn hadoop 128414 Sep 19 2018 ./tezlib/lib/re2j-1.1.jar 297832019 188 -r-xr-xr-x 1 yarn hadoop 190432 Sep 19 2018 ./tezlib/lib/gson-2.2.4.jar 297832020 136 -r-xr-xr-x 1 yarn hadoop 138358 Sep 19 2018 ./tezlib/lib/hadoop-auth-3.1.1.3.0.1.0-187.jar 297832021 296 -r-xr-xr-x 1 yarn hadoop 299508 Jun 22 2018 ./tezlib/lib/nimbus-jose-jwt-4.41.1.jar 297832022 8 -r-xr-xr-x 1 yarn hadoop 4722 Jun 22 2018 ./tezlib/lib/jcip-annotations-1.0-1.jar 297832023 120 -r-xr-xr-x 1 yarn hadoop 120316 Jun 22 2018 ./tezlib/lib/json-smart-2.3.jar 297832024 32 -r-xr-xr-x 1 yarn hadoop 30035 Jun 22 2018 ./tezlib/lib/accessors-smart-1.2.jar 297832025 56 -r-xr-xr-x 1 yarn hadoop 53297 Jun 22 2018 ./tezlib/lib/asm-5.0.4.jar 297832026 788 -r-xr-xr-x 1 yarn hadoop 805053 Sep 19 2018 ./tezlib/lib/zookeeper-3.4.6.3.0.1.0-187.jar 297832027 1300 -r-xr-xr-x 1 yarn hadoop 1330394 Jun 22 2018 ./tezlib/lib/netty-3.10.5.Final.jar 297832028 200 -r-xr-xr-x 1 yarn hadoop 201953 Jun 22 2018 ./tezlib/lib/curator-framework-2.12.0.jar 297832029 2368 -r-xr-xr-x 1 yarn hadoop 2423138 Jun 22 2018 ./tezlib/lib/curator-client-2.12.0.jar 297832030 20 -r-xr-xr-x 1 yarn hadoop 20409 Jun 22 2018 ./tezlib/lib/kerb-simplekdc-1.0.1.jar 297832031 112 -r-xr-xr-x 1 yarn hadoop 113017 Jun 22 2018 ./tezlib/lib/kerb-client-1.0.1.jar 297795809 32 -r-xr-xr-x 1 yarn hadoop 30674 Jun 22 2018 ./tezlib/lib/kerby-config-1.0.1.jar 297795810 224 -r-xr-xr-x 1 yarn hadoop 226672 Jun 22 2018 ./tezlib/lib/kerb-core-1.0.1.jar 297795833 200 -r-xr-xr-x 1 yarn hadoop 204650 Jun 22 2018 ./tezlib/lib/kerby-pkix-1.0.1.jar 297795834 100 -r-xr-xr-x 1 yarn hadoop 102174 Jun 22 2018 ./tezlib/lib/kerby-asn1-1.0.1.jar 297795835 40 -r-xr-xr-x 1 yarn hadoop 40554 Jun 22 2018 ./tezlib/lib/kerby-util-1.0.1.jar 297832288 64 -r-xr-xr-x 1 yarn hadoop 65464 Jun 22 2018 ./tezlib/lib/kerb-common-1.0.1.jar 297832289 116 -r-xr-xr-x 1 yarn hadoop 116120 Jun 22 2018 ./tezlib/lib/kerb-crypto-1.0.1.jar 297832290 36 -r-xr-xr-x 1 yarn hadoop 36708 Jun 22 2018 ./tezlib/lib/kerb-util-1.0.1.jar 297832291 20 -r-xr-xr-x 1 yarn hadoop 18763 Jun 22 2018 ./tezlib/lib/token-provider-1.0.1.jar 297832292 80 -r-xr-xr-x 1 yarn hadoop 80980 Jun 22 2018 ./tezlib/lib/kerb-admin-1.0.1.jar 297832293 84 -r-xr-xr-x 1 yarn hadoop 82756 Jun 22 2018 ./tezlib/lib/kerb-server-1.0.1.jar 297832294 20 -r-xr-xr-x 1 yarn hadoop 20046 Jun 22 2018 ./tezlib/lib/kerb-identity-1.0.1.jar 297832295 32 -r-xr-xr-x 1 yarn hadoop 29134 Jun 22 2018 ./tezlib/lib/kerby-xdr-1.0.1.jar 297832296 280 -r-xr-xr-x 1 yarn hadoop 283598 Jun 22 2018 ./tezlib/lib/curator-recipes-2.12.0.jar 297832297 1468 -r-xr-xr-x 1 yarn hadoop 1502280 Jun 22 2018 ./tezlib/lib/htrace-core4-4.1.0-incubating.jar 297832298 88 -r-xr-xr-x 1 yarn hadoop 87325 Jun 22 2018 ./tezlib/lib/jline-0.9.94.jar 297832299 1312 -r-xr-xr-x 1 yarn hadoop 1342410 Jun 22 2018 ./tezlib/lib/jackson-databind-2.9.5.jar 297832300 316 -r-xr-xr-x 1 yarn hadoop 321590 Jun 22 2018 ./tezlib/lib/jackson-core-2.9.5.jar 297832301 160 -r-xr-xr-x 1 yarn hadoop 161867 Jun 22 2018 ./tezlib/lib/stax2-api-3.1.4.jar 297832302 504 -r-xr-xr-x 1 yarn hadoop 512742 Jun 22 2018 ./tezlib/lib/woodstox-core-5.0.3.jar 297832303 448 -r-xr-xr-x 1 yarn hadoop 456857 Sep 19 2018 ./tezlib/lib/hadoop-aws-3.1.1.3.0.1.0-187.jar 297832304 84200 -r-xr-xr-x 1 yarn hadoop 86220098 Jun 22 2018 ./tezlib/lib/aws-java-sdk-bundle-1.11.271.jar 297832305 544 -r-xr-xr-x 1 yarn hadoop 556241 Jun 22 2018 ./tezlib/lib/netty-codec-http-4.1.17.Final.jar 297832306 308 -r-xr-xr-x 1 yarn hadoop 313647 Jun 22 2018 ./tezlib/lib/netty-codec-4.1.17.Final.jar 297832307 444 -r-xr-xr-x 1 yarn hadoop 452023 Jun 22 2018 ./tezlib/lib/netty-transport-4.1.17.Final.jar 297832308 268 -r-xr-xr-x 1 yarn hadoop 270353 Jun 22 2018 ./tezlib/lib/netty-buffer-4.1.17.Final.jar 297832309 556 -r-xr-xr-x 1 yarn hadoop 565466 Jun 22 2018 ./tezlib/lib/netty-common-4.1.17.Final.jar 297832310 32 -r-xr-xr-x 1 yarn hadoop 32235 Jun 22 2018 ./tezlib/lib/netty-resolver-4.1.17.Final.jar 297832311 368 -r-xr-xr-x 1 yarn hadoop 373661 Jun 22 2018 ./tezlib/lib/netty-handler-4.1.17.Final.jar 297832312 464 -r-xr-xr-x 1 yarn hadoop 472196 Sep 19 2018 ./tezlib/lib/hadoop-azure-3.1.1.3.0.1.0-187.jar 297832313 796 -r-xr-xr-x 1 yarn hadoop 812977 Jun 22 2018 ./tezlib/lib/azure-storage-7.0.0.jar 297832314 12 -r-xr-xr-x 1 yarn hadoop 10288 Jun 22 2018 ./tezlib/lib/azure-keyvault-core-1.0.0.jar 297832315 656 -r-xr-xr-x 1 yarn hadoop 668235 Sep 19 2018 ./tezlib/lib/guice-4.0.jar 297832316 4 -r-xr-xr-x 1 yarn hadoop 2497 Jun 22 2018 ./tezlib/lib/javax.inject-1.jar 297832317 8 -r-xr-xr-x 1 yarn hadoop 4467 Jun 22 2018 ./tezlib/lib/aopalliance-1.0.jar 297832318 40 -r-xr-xr-x 1 yarn hadoop 38707 Jun 22 2018 ./tezlib/lib/jetty-util-ajax-9.3.19.v20170502.jar 297832319 448 -r-xr-xr-x 1 yarn hadoop 456916 Jun 22 2018 ./tezlib/lib/jetty-util-9.3.22.v20171030.jar 297832320 428 -r-xr-xr-x 1 yarn hadoop 437222 Aug 21 2018 ./tezlib/lib/wildfly-openssl-1.0.4.Final.jar 297832321 68 -r-xr-xr-x 1 yarn hadoop 66891 Sep 19 2018 ./tezlib/lib/hadoop-azure-datalake-3.1.1.3.0.1.0-187.jar 297832322 92 -r-xr-xr-x 1 yarn hadoop 92683 Jun 22 2018 ./tezlib/lib/azure-data-lake-store-sdk-2.2.7.jar 297832323 3420 -r-xr-xr-x 1 yarn hadoop 3500453 Sep 19 2018 ./tezlib/lib/gcs-connector-1.9.0.3.0.1.0-187-shaded.jar 297832324 28 -r-xr-xr-x 1 yarn hadoop 26514 Jun 22 2018 ./tezlib/lib/stax-api-1.0.1.jar 297832325 2760 -r-xr-xr-x 1 yarn hadoop 2825438 Sep 19 2018 ./tezlib/lib/hadoop-yarn-common-3.1.1.3.0.1.0-187.jar 297832326 428 -r-xr-xr-x 1 yarn hadoop 436689 Jun 22 2018 ./tezlib/lib/jersey-core-1.19.jar 297832327 48 -r-xr-xr-x 1 yarn hadoop 46367 Jun 22 2018 ./tezlib/lib/jsr311-api-1.1.1.jar 297832328 132 -r-xr-xr-x 1 yarn hadoop 134021 Jun 22 2018 ./tezlib/lib/jersey-client-1.19.jar 297832329 76 -r-xr-xr-x 1 yarn hadoop 76983 Sep 19 2018 ./tezlib/lib/guice-servlet-4.0.jar 297832330 16 -r-xr-xr-x 1 yarn hadoop 16151 Jun 22 2018 ./tezlib/lib/jersey-guice-1.19.jar 297832331 32 -r-xr-xr-x 1 yarn hadoop 32618 Jun 22 2018 ./tezlib/lib/jackson-module-jaxb-annotations-2.9.5.jar 297832332 16 -r-xr-xr-x 1 yarn hadoop 15864 Jun 22 2018 ./tezlib/lib/jackson-jaxrs-json-provider-2.9.5.jar 297832333 32 -r-xr-xr-x 1 yarn hadoop 32373 Jun 22 2018 ./tezlib/lib/jackson-jaxrs-base-2.9.5.jar 297832334 296 -r-xr-xr-x 1 yarn hadoop 300895 Sep 19 2018 ./tezlib/lib/hadoop-yarn-client-3.1.1.3.0.1.0-187.jar 297832335 736 -r-xr-xr-x 1 yarn hadoop 751238 Jun 22 2018 ./tezlib/lib/commons-collections4-4.1.jar 297832336 512 -r-xr-xr-x 1 yarn hadoop 520532 Jun 22 2018 ./tezlib/lib/jetty-server-9.3.22.v20171030.jar 297832337 148 -r-xr-xr-x 1 yarn hadoop 150378 Jun 22 2018 ./tezlib/lib/jetty-http-9.3.22.v20171030.jar 297832338 120 -r-xr-xr-x 1 yarn hadoop 122664 Jun 22 2018 ./tezlib/lib/jetty-io-9.3.22.v20171030.jar 297832339 24 -r-xr-xr-x 1 yarn hadoop 24239 Jun 22 2018 ./tezlib/lib/commons-daemon-1.0.13.jar 297832340 1024 -r-xr-xr-x 1 yarn hadoop 1045744 Jun 22 2018 ./tezlib/lib/leveldbjni-all-1.8.jar 297832341 4872 -r-xr-xr-x 1 yarn hadoop 4986914 Sep 19 2018 ./tezlib/lib/hadoop-hdfs-client-3.1.1.3.0.1.0-187.jar 297832342 324 -r-xr-xr-x 1 yarn hadoop 331034 Jun 22 2018 ./tezlib/lib/okhttp-2.7.5.jar 297832343 68 -r-xr-xr-x 1 yarn hadoop 65928 Jun 22 2018 ./tezlib/lib/okio-1.6.0.jar 297832344 124 -r-xr-xr-x 1 yarn hadoop 124846 Jun 22 2018 ./tezlib/lib/RoaringBitmap-0.4.9.jar 297832345 728 -r-xr-xr-x 1 yarn hadoop 745264 Jun 22 2018 ./tezlib/lib/async-http-client-1.9.40.jar 297832346 1616 -r-xr-xr-x 1 yarn hadoop 1654668 Sep 19 2018 ./tezlib/lib/hadoop-mapreduce-client-core-3.1.1.3.0.1.0-187.jar 297832347 788 -r-xr-xr-x 1 yarn hadoop 803565 Sep 19 2018 ./tezlib/lib/hadoop-mapreduce-client-common-3.1.1.3.0.1.0-187.jar 297832348 80 -r-xr-xr-x 1 yarn hadoop 80462 Sep 19 2018 ./tezlib/lib/hadoop-yarn-server-web-proxy-3.1.1.3.0.1.0-187.jar 297832349 1308 -r-xr-xr-x 1 yarn hadoop 1336861 Sep 19 2018 ./tezlib/lib/hadoop-yarn-server-common-3.1.1.3.0.1.0-187.jar 297832350 224 -r-xr-xr-x 1 yarn hadoop 225710 Sep 19 2018 ./tezlib/lib/hadoop-yarn-registry-3.1.1.3.0.1.0-187.jar 297832351 304 -r-xr-xr-x 1 yarn hadoop 307637 Jun 22 2018 ./tezlib/lib/dnsjava-2.1.7.jar 297832352 56 -r-xr-xr-x 1 yarn hadoop 55236 Jun 22 2018 ./tezlib/lib/geronimo-jcache_1.0_spec-1.0-alpha-1.jar 297832353 1688 -r-xr-xr-x 1 yarn hadoop 1726527 Jun 22 2018 ./tezlib/lib/ehcache-3.3.1.jar 297832354 132 -r-xr-xr-x 1 yarn hadoop 134308 Jun 22 2018 ./tezlib/lib/HikariCP-java7-2.4.12.jar 297832355 776 -r-xr-xr-x 1 yarn hadoop 792442 Jun 22 2018 ./tezlib/lib/mssql-jdbc-6.2.1.jre7.jar 297832356 104 -r-xr-xr-x 1 yarn hadoop 105112 Jun 22 2018 ./tezlib/lib/servlet-api-2.5.jar 297832357 112 -r-xr-xr-x 1 yarn hadoop 111908 Jun 22 2018 ./tezlib/lib/metrics-core-3.1.0.jar 297832358 288 -r-xr-xr-x 1 yarn hadoop 294856 Sep 19 2018 ./tezlib/lib/hadoop-yarn-server-applicationhistoryservice-3.1.1.3.0.1.0-187.jar 297832359 28 -r-xr-xr-x 1 yarn hadoop 28569 Jun 22 2018 ./tezlib/lib/objenesis-1.0.jar 297832360 380 -r-xr-xr-x 1 yarn hadoop 387689 Jun 22 2018 ./tezlib/lib/fst-2.50.jar 297832361 60 -r-xr-xr-x 1 yarn hadoop 58487 Jun 22 2018 ./tezlib/lib/java-util-1.9.0.jar 297832362 76 -r-xr-xr-x 1 yarn hadoop 75232 Jun 22 2018 ./tezlib/lib/json-io-2.5.1.jar 297832363 92 -r-xr-xr-x 1 yarn hadoop 93140 Sep 19 2018 ./tezlib/lib/hadoop-yarn-server-timeline-pluginstorage-3.1.1.3.0.1.0-187.jar 301992118 4 drwxr-xr-x 3 yarn hadoop 4096 Sep 19 2018 ./tezlib/lib/native 301992119 3324 -r-xr-xr-x 2 yarn hadoop 3402313 Sep 19 2018 ./tezlib/lib/native/libnativetask.so 306804541 0 drwxr-xr-x 2 yarn hadoop 94 Sep 19 2018 ./tezlib/lib/native/examples 306804542 964 -r-xr-xr-x 1 yarn hadoop 983078 Sep 19 2018 ./tezlib/lib/native/examples/wordcount-nopipe 306804543 920 -r-xr-xr-x 1 yarn hadoop 938380 Sep 19 2018 ./tezlib/lib/native/examples/wordcount-simple 309092992 924 -r-xr-xr-x 1 yarn hadoop 943052 Sep 19 2018 ./tezlib/lib/native/examples/wordcount-part 309092993 904 -r-xr-xr-x 1 yarn hadoop 924089 Sep 19 2018 ./tezlib/lib/native/examples/pipes-sort 301992120 468 -r-xr-xr-x 1 yarn hadoop 476136 Sep 19 2018 ./tezlib/lib/native/libhadooputils.a 301992121 8396 -r-xr-xr-x 1 yarn hadoop 8594282 Sep 19 2018 ./tezlib/lib/native/libnativetask.a 301992122 24 -r-xr-xr-x 3 yarn hadoop 23800 Jun 10 2014 ./tezlib/lib/native/libsnappy.so.1.1.4 301992123 280 -r-xr-xr-x 2 yarn hadoop 286676 Sep 19 2018 ./tezlib/lib/native/libhdfs.so.0.0.0 301992119 3324 -r-xr-xr-x 2 yarn hadoop 3402313 Sep 19 2018 ./tezlib/lib/native/libnativetask.so.1.0.0 301992124 948 -r-xr-xr-x 2 yarn hadoop 968926 Sep 19 2018 ./tezlib/lib/native/libhadoop.so.1.0.0 301992125 1572 -r-xr-xr-x 1 yarn hadoop 1607384 Sep 19 2018 ./tezlib/lib/native/libhadooppipes.a 301992122 24 -r-xr-xr-x 3 yarn hadoop 23800 Jun 10 2014 ./tezlib/lib/native/libsnappy.so.1 301992122 24 -r-xr-xr-x 3 yarn hadoop 23800 Jun 10 2014 ./tezlib/lib/native/libsnappy.so 301992123 280 -r-xr-xr-x 2 yarn hadoop 286676 Sep 19 2018 ./tezlib/lib/native/libhdfs.so 301992124 948 -r-xr-xr-x 2 yarn hadoop 968926 Sep 19 2018 ./tezlib/lib/native/libhadoop.so 301992126 452 -r-xr-xr-x 1 yarn hadoop 459400 Sep 19 2018 ./tezlib/lib/native/libhdfs.a 301992127 1676 -r-xr-xr-x 1 yarn hadoop 1712408 Sep 19 2018 ./tezlib/lib/native/libhadoop.a 398475942 264 -r-x------ 1 yarn hadoop 269078 Apr 16 12:37 ./hive-hcatalog-core.jar broken symlinks(find -L . -maxdepth 5 -type l -ls): End of LogType:directory.info ******************************************************************************* Container: container_e06_1555415227039_0004_01_000001 on ip-172-31-83-216.ec2.internal_45454_1555418368508 LogAggregationType: AGGREGATED ========================================================================================================== LogType:launch_container.sh LogLastModifiedTime:Tue Apr 16 12:39:28 +0000 2019 LogLength:5409 LogContents: #!/bin/bash set -o pipefail -e export PRELAUNCH_OUT="/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/prelaunch.out" exec >"${PRELAUNCH_OUT}" export PRELAUNCH_ERR="/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/prelaunch.err" exec 2>"${PRELAUNCH_ERR}" echo "Setting up env variables" export JAVA_HOME=${JAVA_HOME:-"/usr/java/default"} export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/usr/hdp/3.0.1.0-187/hadoop/conf"} export HADOOP_YARN_HOME=${HADOOP_YARN_HOME:-"/usr/hdp/3.0.1.0-187/hadoop-yarn"} export HADOOP_HOME=${HADOOP_HOME:-"/usr/hdp/3.0.1.0-187/hadoop"} export PATH=${PATH:-"/usr/sbin:/sbin:/usr/lib/ambari-server/*:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/var/lib/ambari-agent"} export LANG=${LANG:-"en_US.UTF-8"} export HADOOP_TOKEN_FILE_LOCATION="/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/container_tokens" export CONTAINER_ID="container_e06_1555415227039_0004_01_000001" export NM_PORT="45454" export NM_HOST="ip-172-31-83-216.ec2.internal" export NM_HTTP_PORT="8042" export LOCAL_DIRS="/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004" export LOCAL_USER_DIRS="/hadoop/yarn/local/usercache/hive/" export LOG_DIRS="/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001" export USER="hive" export LOGNAME="hive" export HOME="/home/" export PWD="/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001" export JVM_PID="$$" export MALLOC_ARENA_MAX="4" export NM_AUX_SERVICE_timeline_collector="" export NM_AUX_SERVICE_mapreduce_shuffle="AAA0+gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" export NM_AUX_SERVICE_spark2_shuffle="" export TEZ_CLIENT_VERSION="0.9.1.3.0.1.0-187" export APP_SUBMIT_TIME_ENV="1555418242819" export TIMELINE_FLOW_NAME_TAG="HIVE-b7360cc1-1e10-4d86-9994-5f1342034d5d" export LD_LIBRARY_PATH="$PWD:/usr/hdp/3.0.1.0-187/hadoop/lib/native:/usr/hdp/3.0.1.0-187/hadoop/lib/native/Linux-amd64-64:$PWD:/usr/hdp/3.0.1.0-187/hadoop/lib/native:/usr/hdp/3.0.1.0-187/hadoop/lib/native/Linux-amd64-64:$HADOOP_COMMON_HOME/lib/native/" export TIMELINE_FLOW_VERSION_TAG="1" export APPLICATION_WEB_PROXY_BASE="/proxy/application_1555415227039_0004" export CLASSPATH="/usr/hdp/3.0.1.0-187/hadoop/lib/hadoop-lzo-0.6.0.3.0.1.0-187.jar:/etc/hadoop/conf/secure:$PWD:$PWD/*:$PWD/tezlib/*:$PWD/tezlib/lib/*:" export TIMELINE_FLOW_RUN_ID_TAG="1555418242820" echo "Setting up job resources" ln -sf "/hadoop/yarn/local/usercache/hive/filecache/10/hive-exec-3.1.0.3.0.1.0-187-ffcc945cef1d45a4f8c86dab4e04eaea5bb4284dae0788c8b7f4df27141566d9.jar" "hive-exec-3.1.0.3.0.1.0-187-ffcc945cef1d45a4f8c86dab4e04eaea5bb4284dae0788c8b7f4df27141566d9.jar" ln -sf "/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/filecache/10/tez.session.local-resources.pb" "tez.session.local-resources.pb" ln -sf "/hadoop/yarn/local/usercache/hive/filecache/17/TimestampUDF-1.0-SNAPSHOT.jar" "TimestampUDF-1.0-SNAPSHOT.jar" ln -sf "/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/filecache/11/tez-conf.pb" "tez-conf.pb" ln -sf "/hadoop/yarn/local/filecache/10/tez.tar.gz" "tezlib" ln -sf "/hadoop/yarn/local/usercache/hive/filecache/16/hive-hcatalog-core.jar" "hive-hcatalog-core.jar" echo "Copying debugging information" # Creating copy of launch script cp "launch_container.sh" "/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/launch_container.sh" chmod 640 "/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/launch_container.sh" # Determining directory contents echo "ls -l:" 1>"/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/directory.info" ls -l 1>>"/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/directory.info" echo "find -L . -maxdepth 5 -ls:" 1>>"/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/directory.info" find -L . -maxdepth 5 -ls 1>>"/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/directory.info" echo "broken symlinks(find -L . -maxdepth 5 -type l -ls):" 1>>"/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/directory.info" find -L . -maxdepth 5 -type l -ls 1>>"/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/directory.info" echo "Launching container" exec /bin/bash -c "$JAVA_HOME/bin/java -Xmx3276m -Djava.io.tmpdir=$PWD/tmp -server -Djava.net.preferIPv4Stack=true -Dhdp.version=3.0.1.0-187 -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB -Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator -Dlog4j.configuration=tez-container-log4j.properties -Dyarn.app.container.log.dir=/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001 -Dtez.root.logger=INFO,CLA -Dsun.nio.ch.bugLevel='' org.apache.tez.dag.app.DAGAppMaster --session 1>/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/stdout 2>/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/stderr " End of LogType:launch_container.sh ************************************************************************************ End of LogType:prelaunch.err ****************************************************************************** Container: container_e06_1555415227039_0004_01_000001 on ip-172-31-83-216.ec2.internal_45454_1555418368508 LogAggregationType: AGGREGATED ========================================================================================================== LogType:prelaunch.out LogLastModifiedTime:Tue Apr 16 12:39:28 +0000 2019 LogLength:100 LogContents: Setting up env variables Setting up job resources Copying debugging information Launching container End of LogType:prelaunch.out ****************************************************************************** Container: container_e06_1555415227039_0004_01_000001 on ip-172-31-83-216.ec2.internal_45454_1555418368508 LogAggregationType: AGGREGATED ========================================================================================================== LogType:stderr LogLastModifiedTime:Tue Apr 16 12:39:28 +0000 2019 LogLength:118 LogContents: 2019-04-16 12:37:27 Running Dag: dag_1555415227039_0004_1 2019-04-16 12:37:39 Completed Dag: dag_1555415227039_0004_1 End of LogType:stderr *********************************************************************** Container: container_e06_1555415227039_0004_01_000001 on ip-172-31-83-216.ec2.internal_45454_1555418368508 LogAggregationType: AGGREGATED ========================================================================================================== LogType:stdout LogLastModifiedTime:Tue Apr 16 12:39:28 +0000 2019 LogLength:10458 LogContents: 0.289: [GC pause (G1 Evacuation Pause) (young), 0.0037417 secs] [Parallel Time: 2.5 ms, GC Workers: 4] [GC Worker Start (ms): Min: 289.2, Avg: 289.4, Max: 290.0, Diff: 0.8] [Ext Root Scanning (ms): Min: 0.0, Avg: 0.4, Max: 0.5, Diff: 0.5, Sum: 1.4] [Update RS (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [Processed Buffers: Min: 0, Avg: 0.0, Max: 0, Diff: 0, Sum: 0] [Scan RS (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [Code Root Scanning (ms): Min: 0.0, Avg: 0.1, Max: 0.2, Diff: 0.2, Sum: 0.4] [Object Copy (ms): Min: 1.6, Avg: 1.8, Max: 1.9, Diff: 0.3, Sum: 7.3] [Termination (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [Termination Attempts: Min: 1, Avg: 1.8, Max: 2, Diff: 1, Sum: 7] [GC Worker Other (ms): Min: 0.0, Avg: 0.0, Max: 0.1, Diff: 0.1, Sum: 0.1] [GC Worker Total (ms): Min: 1.7, Avg: 2.3, Max: 2.5, Diff: 0.8, Sum: 9.2] [GC Worker End (ms): Min: 291.7, Avg: 291.7, Max: 291.7, Diff: 0.0] [Code Root Fixup: 0.0 ms] [Code Root Purge: 0.0 ms] [Clear CT: 0.1 ms] [Other: 1.1 ms] [Choose CSet: 0.0 ms] [Ref Proc: 0.9 ms] [Ref Enq: 0.0 ms] [Redirty Cards: 0.0 ms] [Humongous Register: 0.0 ms] [Humongous Reclaim: 0.0 ms] [Free CSet: 0.0 ms] [Eden: 14.0M(14.0M)->0.0B(18.0M) Survivors: 0.0B->2048.0K Heap: 14.0M(242.0M)->2823.5K(242.0M)] [Times: user=0.01 sys=0.00, real=0.01 secs] 0.624: [GC pause (G1 Evacuation Pause) (young), 0.0122430 secs] [Parallel Time: 8.6 ms, GC Workers: 4] [GC Worker Start (ms): Min: 623.7, Avg: 627.9, Max: 632.2, Diff: 8.5] [Ext Root Scanning (ms): Min: 0.0, Avg: 0.6, Max: 1.2, Diff: 1.2, Sum: 2.3] [Update RS (ms): Min: 0.0, Avg: 0.1, Max: 0.2, Diff: 0.2, Sum: 0.3] [Processed Buffers: Min: 0, Avg: 0.5, Max: 1, Diff: 1, Sum: 2] [Scan RS (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [Code Root Scanning (ms): Min: 0.0, Avg: 0.2, Max: 0.6, Diff: 0.6, Sum: 0.8] [Object Copy (ms): Min: 0.0, Avg: 2.6, Max: 5.3, Diff: 5.3, Sum: 10.5] [Termination (ms): Min: 0.0, Avg: 0.8, Max: 1.3, Diff: 1.3, Sum: 3.3] [Termination Attempts: Min: 1, Avg: 1.0, Max: 1, Diff: 0, Sum: 4] [GC Worker Other (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [GC Worker Total (ms): Min: 0.0, Avg: 4.3, Max: 8.6, Diff: 8.6, Sum: 17.3] [GC Worker End (ms): Min: 632.2, Avg: 632.2, Max: 632.3, Diff: 0.0] [Code Root Fixup: 0.0 ms] [Code Root Purge: 0.0 ms] [Clear CT: 0.1 ms] [Other: 3.5 ms] [Choose CSet: 0.0 ms] [Ref Proc: 3.3 ms] [Ref Enq: 0.0 ms] [Redirty Cards: 0.1 ms] [Humongous Register: 0.0 ms] [Humongous Reclaim: 0.0 ms] [Free CSet: 0.1 ms] [Eden: 18.0M(18.0M)->0.0B(142.0M) Survivors: 2048.0K->3072.0K Heap: 20.8M(242.0M)->6563.0K(242.0M)] [Times: user=0.03 sys=0.01, real=0.01 secs] 2.049: [GC pause (G1 Evacuation Pause) (young), 0.0315424 secs] [Parallel Time: 21.8 ms, GC Workers: 4] [GC Worker Start (ms): Min: 2049.6, Avg: 2060.2, Max: 2070.7, Diff: 21.1] [Ext Root Scanning (ms): Min: 0.0, Avg: 0.9, Max: 3.8, Diff: 3.8, Sum: 3.8] [Update RS (ms): Min: 0.0, Avg: 0.3, Max: 1.4, Diff: 1.4, Sum: 1.4] [Processed Buffers: Min: 0, Avg: 2.2, Max: 9, Diff: 9, Sum: 9] [Scan RS (ms): Min: 0.0, Avg: 0.1, Max: 0.2, Diff: 0.2, Sum: 0.3] [Code Root Scanning (ms): Min: 0.0, Avg: 0.1, Max: 0.6, Diff: 0.6, Sum: 0.6] [Object Copy (ms): Min: 0.0, Avg: 7.3, Max: 13.2, Diff: 13.2, Sum: 29.3] [Termination (ms): Min: 0.0, Avg: 1.9, Max: 2.9, Diff: 2.9, Sum: 7.6] [Termination Attempts: Min: 1, Avg: 1.8, Max: 4, Diff: 3, Sum: 7] [GC Worker Other (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [GC Worker Total (ms): Min: 0.0, Avg: 10.7, Max: 21.3, Diff: 21.3, Sum: 42.9] [GC Worker End (ms): Min: 2070.7, Avg: 2070.9, Max: 2071.3, Diff: 0.6] [Code Root Fixup: 0.0 ms] [Code Root Purge: 0.0 ms] [Clear CT: 0.1 ms] [Other: 9.6 ms] [Choose CSet: 0.0 ms] [Ref Proc: 9.2 ms] [Ref Enq: 0.0 ms] [Redirty Cards: 0.0 ms] [Humongous Register: 0.0 ms] [Humongous Reclaim: 0.0 ms] [Free CSet: 0.1 ms] [Eden: 142.0M(142.0M)->0.0B(129.0M) Survivors: 3072.0K->16.0M Heap: 148.4M(242.0M)->19.3M(242.0M)] [Times: user=0.04 sys=0.01, real=0.04 secs] 2.085: [GC pause (Metadata GC Threshold) (young) (initial-mark), 0.0150151 secs] [Parallel Time: 13.5 ms, GC Workers: 4] [GC Worker Start (ms): Min: 2084.8, Avg: 2086.9, Max: 2087.6, Diff: 2.8] [Ext Root Scanning (ms): Min: 0.1, Avg: 7.3, Max: 10.6, Diff: 10.5, Sum: 29.0] [Update RS (ms): Min: 0.0, Avg: 0.1, Max: 0.4, Diff: 0.4, Sum: 0.4] [Processed Buffers: Min: 0, Avg: 1.5, Max: 6, Diff: 6, Sum: 6] [Scan RS (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [Code Root Scanning (ms): Min: 0.0, Avg: 0.1, Max: 0.5, Diff: 0.5, Sum: 0.5] [Object Copy (ms): Min: 0.0, Avg: 3.0, Max: 8.2, Diff: 8.2, Sum: 12.0] [Termination (ms): Min: 0.0, Avg: 0.8, Max: 1.3, Diff: 1.3, Sum: 3.3] [Termination Attempts: Min: 1, Avg: 1.0, Max: 1, Diff: 0, Sum: 4] [GC Worker Other (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [GC Worker Total (ms): Min: 10.6, Avg: 11.3, Max: 13.4, Diff: 2.8, Sum: 45.3] [GC Worker End (ms): Min: 2098.2, Avg: 2098.2, Max: 2098.3, Diff: 0.0] [Code Root Fixup: 0.0 ms] [Code Root Purge: 0.0 ms] [Clear CT: 0.0 ms] [Other: 1.5 ms] [Choose CSet: 0.0 ms] [Ref Proc: 1.3 ms] [Ref Enq: 0.0 ms] [Redirty Cards: 0.0 ms] [Humongous Register: 0.0 ms] [Humongous Reclaim: 0.0 ms] [Free CSet: 0.0 ms] [Eden: 1024.0K(129.0M)->0.0B(139.0M) Survivors: 16.0M->6144.0K Heap: 20.3M(242.0M)->9123.0K(242.0M)] [Times: user=0.02 sys=0.00, real=0.01 secs] 2.105: [GC concurrent-root-region-scan-start] 2.112: [GC concurrent-root-region-scan-end, 0.0070444 secs] 2.112: [GC concurrent-mark-start] 2.116: [GC concurrent-mark-end, 0.0040919 secs] 2.133: [GC remark 2.133: [Finalize Marking, 0.0058027 secs] 2.139: [GC ref-proc, 0.0001225 secs] 2.139: [Unloading, 0.0046196 secs], 0.0107140 secs] [Times: user=0.02 sys=0.00, real=0.01 secs] 2.150: [GC cleanup 11M->11M(242M), 0.0004200 secs] [Times: user=0.00 sys=0.00, real=0.00 secs] 2.152: [GC concurrent-cleanup-start] 2.152: [GC concurrent-cleanup-end, 0.0000182 secs] 3.371: [GC pause (G1 Evacuation Pause) (young) (initial-mark), 0.0241658 secs] [Parallel Time: 18.5 ms, GC Workers: 4] [GC Worker Start (ms): Min: 3371.7, Avg: 3372.8, Max: 3373.9, Diff: 2.2] [Ext Root Scanning (ms): Min: 2.4, Avg: 5.8, Max: 14.4, Diff: 12.0, Sum: 23.2] [Update RS (ms): Min: 0.0, Avg: 0.1, Max: 0.5, Diff: 0.5, Sum: 0.5] [Processed Buffers: Min: 0, Avg: 2.8, Max: 11, Diff: 11, Sum: 11] [Scan RS (ms): Min: 0.0, Avg: 0.0, Max: 0.1, Diff: 0.1, Sum: 0.1] [Code Root Scanning (ms): Min: 0.0, Avg: 0.1, Max: 0.6, Diff: 0.6, Sum: 0.6] [Object Copy (ms): Min: 3.4, Avg: 10.8, Max: 13.8, Diff: 10.4, Sum: 43.1] [Termination (ms): Min: 0.0, Avg: 0.5, Max: 0.6, Diff: 0.6, Sum: 1.9] [Termination Attempts: Min: 1, Avg: 1.8, Max: 4, Diff: 3, Sum: 7] [GC Worker Other (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [GC Worker Total (ms): Min: 16.2, Avg: 17.4, Max: 18.4, Diff: 2.2, Sum: 69.5] [GC Worker End (ms): Min: 3390.1, Avg: 3390.1, Max: 3390.1, Diff: 0.0] [Code Root Fixup: 0.0 ms] [Code Root Purge: 0.0 ms] [Clear CT: 0.1 ms] [Other: 5.5 ms] [Choose CSet: 0.0 ms] [Ref Proc: 5.1 ms] [Ref Enq: 0.0 ms] [Redirty Cards: 0.0 ms] [Humongous Register: 0.0 ms] [Humongous Reclaim: 0.0 ms] [Free CSet: 0.1 ms] [Eden: 139.0M(139.0M)->0.0B(126.0M) Survivors: 6144.0K->19.0M Heap: 147.5M(242.0M)->23.1M(242.0M)] [Times: user=0.06 sys=0.01, real=0.02 secs] 3.396: [GC concurrent-root-region-scan-start] 3.415: [GC concurrent-root-region-scan-end, 0.0195991 secs] 3.415: [GC concurrent-mark-start] 3.420: [GC concurrent-mark-end, 0.0042707 secs] 3.424: [GC remark 3.424: [Finalize Marking, 0.0007388 secs] 3.425: [GC ref-proc, 0.0001189 secs] 3.425: [Unloading, 0.0053899 secs], 0.0063908 secs] [Times: user=0.02 sys=0.00, real=0.00 secs] 3.430: [GC cleanup 25M->25M(242M), 0.0004288 secs] [Times: user=0.00 sys=0.00, real=0.01 secs] 2019-04-16 12:37:27 Running Dag: dag_1555415227039_0004_1 5.456: [GC pause (G1 Evacuation Pause) (young), 0.0264347 secs] [Parallel Time: 18.0 ms, GC Workers: 4] [GC Worker Start (ms): Min: 5455.8, Avg: 5455.8, Max: 5455.9, Diff: 0.2] [Ext Root Scanning (ms): Min: 1.1, Avg: 1.8, Max: 3.4, Diff: 2.3, Sum: 7.1] [Update RS (ms): Min: 0.0, Avg: 0.2, Max: 0.4, Diff: 0.4, Sum: 0.9] [Processed Buffers: Min: 0, Avg: 4.5, Max: 14, Diff: 14, Sum: 18] [Scan RS (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.1] [Code Root Scanning (ms): Min: 0.0, Avg: 0.2, Max: 0.6, Diff: 0.6, Sum: 1.0] [Object Copy (ms): Min: 14.4, Avg: 15.6, Max: 16.1, Diff: 1.6, Sum: 62.2] [Termination (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [Termination Attempts: Min: 1, Avg: 3.0, Max: 5, Diff: 4, Sum: 12] [GC Worker Other (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [GC Worker Total (ms): Min: 17.7, Avg: 17.8, Max: 17.9, Diff: 0.2, Sum: 71.4] [GC Worker End (ms): Min: 5473.7, Avg: 5473.7, Max: 5473.7, Diff: 0.0] [Code Root Fixup: 0.0 ms] [Code Root Purge: 0.0 ms] [Clear CT: 0.1 ms] [Other: 8.4 ms] [Choose CSet: 0.0 ms] [Ref Proc: 8.0 ms] [Ref Enq: 0.0 ms] [Redirty Cards: 0.0 ms] [Humongous Register: 0.0 ms] [Humongous Reclaim: 0.0 ms] [Free CSet: 0.1 ms] [Eden: 126.0M(126.0M)->0.0B(126.0M) Survivors: 19.0M->19.0M Heap: 149.1M(242.0M)->36.5M(242.0M)] [Times: user=0.07 sys=0.00, real=0.03 secs] 2019-04-16 12:37:39 Completed Dag: dag_1555415227039_0004_1 Heap garbage-first heap total 247808K, used 75305K [0x00000006f3400000, 0x00000006f3500790, 0x00000007c0000000) region size 1024K, 57 young (58368K), 19 survivors (19456K) Metaspace used 49683K, capacity 50093K, committed 50508K, reserved 1093632K class space used 5714K, capacity 5848K, committed 5964K, reserved 1048576K End of LogType:stdout *********************************************************************** Container: container_e06_1555415227039_0004_01_000001 on ip-172-31-83-216.ec2.internal_45454_1555418368508 LogAggregationType: AGGREGATED ========================================================================================================== LogType:syslog LogLastModifiedTime:Tue Apr 16 12:39:28 +0000 2019 LogLength:42188 LogContents: 2019-04-16 12:37:23,668 [INFO] [main] |app.DAGAppMaster|: Creating DAGAppMaster for applicationId=application_1555415227039_0004, attemptNum=1, AMContainerId=container_e06_1555415227039_0004_01_000001, jvmPid=20277, userFromEnv=hive, cliSessionOption=true, pwd=/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001, localDirs=/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004, logDirs=/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001 2019-04-16 12:37:23,768 [INFO] [main] |Configuration.deprecation|: yarn.resourcemanager.zk-timeout-ms is deprecated. Instead, use hadoop.zk.timeout-ms 2019-04-16 12:37:23,772 [INFO] [main] |Configuration.deprecation|: yarn.resourcemanager.zk-retry-interval-ms is deprecated. Instead, use hadoop.zk.retry-interval-ms 2019-04-16 12:37:23,773 [INFO] [main] |Configuration.deprecation|: yarn.resourcemanager.display.per-user-apps is deprecated. Instead, use yarn.webapp.filter-entity-list-by-user 2019-04-16 12:37:23,774 [INFO] [main] |Configuration.deprecation|: yarn.resourcemanager.zk-num-retries is deprecated. Instead, use hadoop.zk.num-retries 2019-04-16 12:37:23,776 [INFO] [main] |Configuration.deprecation|: yarn.resourcemanager.zk-address is deprecated. Instead, use hadoop.zk.address 2019-04-16 12:37:23,779 [INFO] [main] |Configuration.deprecation|: yarn.resourcemanager.zk-acl is deprecated. Instead, use hadoop.zk.acl 2019-04-16 12:37:23,780 [INFO] [main] |Configuration.deprecation|: yarn.resourcemanager.system-metrics-publisher.enabled is deprecated. Instead, use yarn.system-metrics-publisher.enabled 2019-04-16 12:37:23,970 [INFO] [main] |security.SecurityUtil|: Updating Configuration 2019-04-16 12:37:24,369 [INFO] [main] |app.DAGAppMaster|: Created DAGAppMaster for application appattempt_1555415227039_0004_000001, versionInfo=[ component=tez-dag, version=0.9.1.3.0.1.0-187, revision=a546e73be6ee94bb0e672f5d361f7408dc1bf418, SCM-URL=scm:git:https://git-wip-us.apache.org/repos/asf/tez.git, buildTime=2018-09-19T10:13:49Z ] 2019-04-16 12:37:24,439 [INFO] [main] |app.DAGAppMaster|: /************************************************************ [system properties] os.name: Linux os.version: 3.10.0-693.5.2.el7.x86_64 java.home: /usr/java/jdk1.8.0_161/jre java.runtime.version: 1.8.0_161-b12 java.vendor: Oracle Corporation java.version: 1.8.0_161 java.vm.name: Java HotSpot(TM) 64-Bit Server VM java.class.path: /usr/hdp/3.0.1.0-187/hadoop/lib/hadoop-lzo-0.6.0.3.0.1.0-187.jar:/etc/hadoop/conf/secure:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/hive-exec-3.1.0.3.0.1.0-187-ffcc945cef1d45a4f8c86dab4e04eaea5bb4284dae0788c8b7f4df27141566d9.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/TimestampUDF-1.0-SNAPSHOT.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/hive-hcatalog-core.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/hadoop-shim-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/tez-api-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/tez-common-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/tez-runtime-internals-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/tez-runtime-library-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/tez-mapreduce-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/tez-examples-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/tez-dag-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/tez-tests-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/tez-ext-service-tests-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/tez-protobuf-history-plugin-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/tez-yarn-timeline-history-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/tez-yarn-timeline-history-with-acls-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/tez-yarn-timeline-cache-plugin-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/tez-yarn-timeline-history-with-fs-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/tez-history-parser-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/tez-job-analyzer-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/tez-javadoc-tools-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/hadoop-shim-2.8-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/slf4j-api-1.7.10.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/hadoop-yarn-api-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/commons-lang-2.6.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/guava-11.0.2.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jsr305-3.0.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/commons-logging-1.1.3.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jaxb-api-2.2.11.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/protobuf-java-2.5.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jackson-annotations-2.9.5.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/hadoop-common-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/hadoop-annotations-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/commons-cli-1.2.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/commons-math3-3.1.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/httpclient-4.5.2.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/httpcore-4.4.4.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/commons-codec-1.4.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/commons-io-2.4.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/commons-net-3.6.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/commons-collections-3.2.2.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/javax.servlet-api-3.1.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jetty-servlet-9.3.19.v20170502.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jetty-security-9.3.19.v20170502.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jetty-webapp-9.3.19.v20170502.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jetty-xml-9.3.19.v20170502.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jersey-servlet-1.19.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jersey-json-1.19.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jettison-1.3.4.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jaxb-impl-2.2.3-1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jackson-core-asl-1.9.2.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jackson-mapper-asl-1.9.2.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jackson-jaxrs-1.9.2.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jackson-xc-1.9.2.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/log4j-1.2.17.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/commons-beanutils-1.9.3.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/commons-configuration2-2.1.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/commons-lang3-3.4.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/slf4j-log4j12-1.7.10.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/avro-1.7.7.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/paranamer-2.3.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/snappy-java-1.0.5.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/commons-compress-1.4.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/xz-1.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/re2j-1.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/gson-2.2.4.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/hadoop-auth-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/nimbus-jose-jwt-4.41.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jcip-annotations-1.0-1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/json-smart-2.3.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/accessors-smart-1.2.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/asm-5.0.4.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/zookeeper-3.4.6.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/netty-3.10.5.Final.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/curator-framework-2.12.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/curator-client-2.12.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/kerb-simplekdc-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/kerb-client-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/kerby-config-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/kerb-core-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/kerby-pkix-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/kerby-asn1-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/kerby-util-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/kerb-common-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/kerb-crypto-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/kerb-util-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/token-provider-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/kerb-admin-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/kerb-server-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/kerb-identity-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/kerby-xdr-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/curator-recipes-2.12.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/htrace-core4-4.1.0-incubating.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jline-0.9.94.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jackson-databind-2.9.5.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jackson-core-2.9.5.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/stax2-api-3.1.4.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/woodstox-core-5.0.3.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/hadoop-aws-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/aws-java-sdk-bundle-1.11.271.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/netty-codec-http-4.1.17.Final.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/netty-codec-4.1.17.Final.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/netty-transport-4.1.17.Final.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/netty-buffer-4.1.17.Final.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/netty-common-4.1.17.Final.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/netty-resolver-4.1.17.Final.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/netty-handler-4.1.17.Final.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/hadoop-azure-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/azure-storage-7.0.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/azure-keyvault-core-1.0.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/guice-4.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/javax.inject-1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/aopalliance-1.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jetty-util-ajax-9.3.19.v20170502.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jetty-util-9.3.22.v20171030.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/wildfly-openssl-1.0.4.Final.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/hadoop-azure-datalake-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/azure-data-lake-store-sdk-2.2.7.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/gcs-connector-1.9.0.3.0.1.0-187-shaded.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/stax-api-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/hadoop-yarn-common-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jersey-core-1.19.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jsr311-api-1.1.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jersey-client-1.19.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/guice-servlet-4.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jersey-guice-1.19.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jackson-module-jaxb-annotations-2.9.5.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jackson-jaxrs-json-provider-2.9.5.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jackson-jaxrs-base-2.9.5.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/hadoop-yarn-client-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/commons-collections4-4.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jetty-server-9.3.22.v20171030.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jetty-http-9.3.22.v20171030.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/jetty-io-9.3.22.v20171030.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/commons-daemon-1.0.13.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/leveldbjni-all-1.8.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/hadoop-hdfs-client-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/okhttp-2.7.5.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/okio-1.6.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/RoaringBitmap-0.4.9.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/async-http-client-1.9.40.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/hadoop-mapreduce-client-core-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/hadoop-mapreduce-client-common-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/hadoop-yarn-server-web-proxy-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/hadoop-yarn-server-common-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/hadoop-yarn-registry-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/dnsjava-2.1.7.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/geronimo-jcache_1.0_spec-1.0-alpha-1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/ehcache-3.3.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/HikariCP-java7-2.4.12.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/mssql-jdbc-6.2.1.jre7.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/servlet-api-2.5.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/metrics-core-3.1.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/hadoop-yarn-server-applicationhistoryservice-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/objenesis-1.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/fst-2.50.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/java-util-1.9.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/json-io-2.5.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tezlib/lib/hadoop-yarn-server-timeline-pluginstorage-3.1.1.3.0.1.0-187.jar: java.io.tmpdir: /hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tmp user.dir: /hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001 user.name: yarn ************************************************************/ 2019-04-16 12:37:24,454 [INFO] [main] |shim.HadoopShimsLoader|: Trying to locate HadoopShimProvider for hadoopVersion=3.1.1.3.0.1.0-187, majorVersion=3, minorVersion=1 2019-04-16 12:37:24,457 [INFO] [main] |shim.HadoopShimsLoader|: Picked HadoopShim org.apache.tez.hadoop.shim.DefaultHadoopShim, providerName=null, overrideProviderViaConfig=null, hadoopVersion=3.1.1.3.0.1.0-187, majorVersion=3, minorVersion=1 2019-04-16 12:37:24,520 [INFO] [main] |app.DAGAppMaster|: AM Level configured TaskSchedulers: [0:TezYarn:null],[1:TezUber:null] 2019-04-16 12:37:24,521 [INFO] [main] |app.DAGAppMaster|: AM Level configured ContainerLaunchers: [0:TezYarn:null],[1:TezUber:null] 2019-04-16 12:37:24,521 [INFO] [main] |app.DAGAppMaster|: AM Level configured TaskCommunicators: [0:TezYarn:null],[1:TezUber:null] 2019-04-16 12:37:24,521 [INFO] [main] |app.DAGAppMaster|: Comparing client version with AM version, clientVersion=0.9.1.3.0.1.0-187, AMVersion=0.9.1.3.0.1.0-187 2019-04-16 12:37:25,405 [INFO] [main] |app.TaskCommunicatorManager|: Creating Default Task Communicator 2019-04-16 12:37:25,411 [INFO] [main] |Configuration.deprecation|: mapred.input.dir.recursive is deprecated. Instead, use mapreduce.input.fileinputformat.input.dir.recursive 2019-04-16 12:37:25,415 [INFO] [main] |Configuration.deprecation|: io.bytes.per.checksum is deprecated. Instead, use dfs.bytes-per-checksum 2019-04-16 12:37:25,416 [INFO] [main] |Configuration.deprecation|: yarn.resourcemanager.system-metrics-publisher.enabled is deprecated. Instead, use yarn.system-metrics-publisher.enabled 2019-04-16 12:37:25,416 [INFO] [main] |app.TaskCommunicatorManager|: Creating Default Local Task Communicator 2019-04-16 12:37:25,426 [INFO] [main] |common.AsyncDispatcher|: Registering class org.apache.tez.dag.app.rm.container.AMContainerEventType for class org.apache.tez.dag.app.rm.container.AMContainerMap 2019-04-16 12:37:25,428 [INFO] [main] |common.AsyncDispatcher|: Registering class org.apache.tez.dag.app.rm.node.AMNodeEventType for class org.apache.tez.dag.app.rm.node.AMNodeTracker 2019-04-16 12:37:25,429 [INFO] [main] |common.AsyncDispatcher|: Registering class org.apache.tez.dag.app.dag.event.DAGAppMasterEventType for class org.apache.tez.dag.app.DAGAppMaster$DAGAppMasterEventHandler 2019-04-16 12:37:25,429 [INFO] [main] |common.AsyncDispatcher|: Registering class org.apache.tez.dag.app.dag.event.DAGEventType for class org.apache.tez.dag.app.DAGAppMaster$DagEventDispatcher 2019-04-16 12:37:25,430 [INFO] [main] |common.AsyncDispatcher|: Registering class org.apache.tez.dag.app.dag.event.VertexEventType for class org.apache.tez.dag.app.DAGAppMaster$VertexEventDispatcher 2019-04-16 12:37:25,430 [INFO] [main] |app.DAGAppMaster|: Using concurrent dispatcher: false 2019-04-16 12:37:25,430 [INFO] [main] |common.AsyncDispatcher|: Registering class org.apache.tez.dag.app.dag.event.TaskEventType for class org.apache.tez.dag.app.DAGAppMaster$TaskEventDispatcher 2019-04-16 12:37:25,431 [INFO] [main] |common.AsyncDispatcher|: Registering class org.apache.tez.dag.app.dag.event.TaskAttemptEventType for class org.apache.tez.dag.app.DAGAppMaster$TaskAttemptEventDispatcher 2019-04-16 12:37:25,431 [INFO] [main] |common.AsyncDispatcher|: Registering class org.apache.tez.dag.app.dag.event.SpeculatorEventType for independent dispatch using: class org.apache.tez.dag.app.DAGAppMaster$SpeculatorEventHandler 2019-04-16 12:37:25,432 [INFO] [main] |common.AsyncDispatcher|: Registering class org.apache.tez.dag.app.dag.event.SpeculatorEventType for class org.apache.tez.dag.app.DAGAppMaster$SpeculatorEventHandler 2019-04-16 12:37:25,461 [INFO] [main] |common.AsyncDispatcher|: Registering class org.apache.tez.dag.app.rm.AMSchedulerEventType for class org.apache.tez.dag.app.rm.TaskSchedulerManager 2019-04-16 12:37:25,464 [INFO] [main] |launcher.TezContainerLauncherImpl|: Creating DefaultContainerLauncher 2019-04-16 12:37:25,472 [INFO] [main] |launcher.TezContainerLauncherImpl|: Upper limit on the thread pool size is 500 2019-04-16 12:37:25,473 [INFO] [main] |launcher.TezContainerLauncherImpl|: Creating LocalContainerLauncher 2019-04-16 12:37:25,486 [INFO] [main] |common.AsyncDispatcher|: Registering class org.apache.tez.dag.app.rm.ContainerLauncherEventType for class org.apache.tez.dag.app.launcher.ContainerLauncherManager 2019-04-16 12:37:25,541 [INFO] [main] |node.AMNodeTracker|: blacklistDisablePercent is 33, blacklistingEnabled: true, maxTaskFailuresPerNode: 10, nodeUpdatesRescheduleEnabled: false 2019-04-16 12:37:25,543 [ERROR] [main] |web.WebUIService|: Tez UI History URL is not set 2019-04-16 12:37:25,543 [INFO] [main] |history.HistoryEventHandler|: Initializing HistoryEventHandler withrecoveryEnabled=true, historyServiceClassName=org.apache.tez.dag.history.logging.proto.ProtoHistoryLoggingService 2019-04-16 12:37:25,651 [INFO] [main] |proto.ProtoHistoryLoggingService|: Initing ProtoHistoryLoggingService 2019-04-16 12:37:25,651 [INFO] [main] |proto.ProtoHistoryLoggingService|: Inited ProtoHistoryLoggingService 2019-04-16 12:37:25,652 [INFO] [main] |recovery.RecoveryService|: RecoveryService initialized with recoveryPath=hdfs://ip-172-31-69-167.ec2.internal:8020/tmp/hive/hive/_tez_session_dir/b7360cc1-1e10-4d86-9994-5f1342034d5d/.tez/application_1555415227039_0004/recovery/1, bufferSize(bytes)=8192, flushInterval(s)=30, maxUnflushedEvents=100 2019-04-16 12:37:25,653 [INFO] [main] |HistoryEventHandler.criticalEvents|: [HISTORY][DAG:N/A][Event:APP_LAUNCHED]: applicationId=application_1555415227039_0004, appSubmitTime=1555418242819, launchTime=1555418244366 2019-04-16 12:37:25,654 [INFO] [main] |HistoryEventHandler.criticalEvents|: [HISTORY][DAG:N/A][Event:AM_LAUNCHED]: appAttemptId=appattempt_1555415227039_0004_000001, appSubmitTime=1555418242819, launchTime=1555418244366 2019-04-16 12:37:25,709 [INFO] [ServiceThread:org.apache.tez.dag.app.TaskCommunicatorManager] |ipc.CallQueueManager|: Using callQueue: class java.util.concurrent.LinkedBlockingQueue queueCapacity: 3000 scheduler: class org.apache.hadoop.ipc.DefaultRpcScheduler 2019-04-16 12:37:25,709 [INFO] [ServiceThread:DAGClientRPCServer] |ipc.CallQueueManager|: Using callQueue: class java.util.concurrent.LinkedBlockingQueue queueCapacity: 200 scheduler: class org.apache.hadoop.ipc.DefaultRpcScheduler 2019-04-16 12:37:25,722 [INFO] [ServiceThread:org.apache.tez.dag.history.HistoryEventHandler] |proto.ProtoHistoryLoggingService|: Starting ProtoHistoryLoggingService 2019-04-16 12:37:25,734 [INFO] [ServiceThread:org.apache.tez.dag.app.web.WebUIService] |util.log|: Logging initialized @2520ms 2019-04-16 12:37:25,736 [INFO] [Socket Reader #1 for port 45483] |ipc.Server|: Starting Socket Reader #1 for port 45483 2019-04-16 12:37:25,750 [INFO] [Socket Reader #1 for port 38482] |ipc.Server|: Starting Socket Reader #1 for port 38482 2019-04-16 12:37:25,769 [INFO] [IPC Server Responder] |ipc.Server|: IPC Server Responder: starting 2019-04-16 12:37:25,770 [INFO] [IPC Server listener on 45483] |ipc.Server|: IPC Server listener on 45483: starting 2019-04-16 12:37:25,806 [INFO] [ServiceThread:org.apache.tez.dag.app.TaskCommunicatorManager] |app.TezTaskCommunicatorImpl|: Instantiated TezTaskCommunicator RPC at ip-172-31-83-216.ec2.internal/172.31.83.216:45483 2019-04-16 12:37:25,807 [INFO] [IPC Server Responder] |ipc.Server|: IPC Server Responder: starting 2019-04-16 12:37:25,812 [INFO] [IPC Server listener on 38482] |ipc.Server|: IPC Server listener on 38482: starting 2019-04-16 12:37:25,856 [INFO] [ServiceThread:DAGClientRPCServer] |client.DAGClientServer|: Instantiated DAGClientRPCServer at ip-172-31-83-216.ec2.internal/172.31.83.216:38482 2019-04-16 12:37:25,918 [INFO] [ServiceThread:org.apache.tez.dag.app.web.WebUIService] |server.AuthenticationFilter|: Unable to initialize FileSignerSecretProvider, falling back to use random secrets. 2019-04-16 12:37:25,926 [INFO] [ServiceThread:org.apache.tez.dag.app.web.WebUIService] |http.HttpRequestLog|: Http request log for http.requests. is not defined 2019-04-16 12:37:25,929 [INFO] [ServiceThread:org.apache.tez.dag.app.web.WebUIService] |http.HttpServer2|: Added global filter 'safety' (class=org.apache.hadoop.http.HttpServer2$QuotingInputFilter) 2019-04-16 12:37:25,932 [INFO] [ServiceThread:org.apache.tez.dag.history.HistoryEventHandler] |zlib.ZlibFactory|: Successfully loaded & initialized native-zlib library 2019-04-16 12:37:25,933 [INFO] [ServiceThread:org.apache.tez.dag.history.HistoryEventHandler] |compress.CodecPool|: Got brand-new compressor [.deflate] 2019-04-16 12:37:25,947 [INFO] [ServiceThread:org.apache.tez.dag.app.web.WebUIService] |http.HttpServer2|: Added filter AM_PROXY_FILTER (class=org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter) to context 2019-04-16 12:37:25,947 [INFO] [ServiceThread:org.apache.tez.dag.app.web.WebUIService] |http.HttpServer2|: Added filter AM_PROXY_FILTER (class=org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter) to context static 2019-04-16 12:37:25,948 [INFO] [ServiceThread:org.apache.tez.dag.history.HistoryEventHandler] |proto.ProtoHistoryLoggingService|: Started ProtoHistoryLoggingService 2019-04-16 12:37:25,952 [INFO] [ServiceThread:org.apache.tez.dag.app.web.WebUIService] |http.HttpServer2|: adding path spec: /* 2019-04-16 12:37:26,335 [INFO] [ServiceThread:org.apache.tez.dag.app.web.WebUIService] |webapp.WebApps|: Registered webapp guice modules 2019-04-16 12:37:26,336 [INFO] [ServiceThread:org.apache.tez.dag.app.web.WebUIService] |http.HttpServer2|: Jetty bound to port 33597 2019-04-16 12:37:26,339 [INFO] [ServiceThread:org.apache.tez.dag.app.web.WebUIService] |server.Server|: jetty-9.3.22.v20171030 2019-04-16 12:37:26,384 [INFO] [ServiceThread:org.apache.tez.dag.app.web.WebUIService] |handler.ContextHandler|: Started o.e.j.s.ServletContextHandler@3b5f737{/static,jar:file:/hadoop/yarn/local/filecache/10/tez.tar.gz/lib/hadoop-yarn-common-3.1.1.3.0.1.0-187.jar!/webapps/static,AVAILABLE} 2019-04-16 12:37:26,542 [INFO] [ServiceThread:org.apache.tez.dag.app.web.WebUIService] |handler.ContextHandler|: Started o.e.j.w.WebAppContext@1aa1bf6e{/,file:///hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/tmp/jetty-0.0.0.0-33597-webapps-_-any-2799289532710293711.dir/webapp/,AVAILABLE}{/} 2019-04-16 12:37:26,547 [INFO] [ServiceThread:org.apache.tez.dag.app.web.WebUIService] |server.AbstractConnector|: Started ServerConnector@3be5a72a{HTTP/1.1,[http/1.1]}{0.0.0.0:33597} 2019-04-16 12:37:26,547 [INFO] [ServiceThread:org.apache.tez.dag.app.web.WebUIService] |server.Server|: Started @3345ms 2019-04-16 12:37:26,547 [INFO] [ServiceThread:org.apache.tez.dag.app.web.WebUIService] |webapp.WebApps|: Web app started at 33597 2019-04-16 12:37:26,548 [INFO] [ServiceThread:org.apache.tez.dag.app.web.WebUIService] |web.WebUIService|: Instantiated WebUIService at http://ip-172-31-83-216.ec2.internal:33597/ui/ 2019-04-16 12:37:26,549 [INFO] [ServiceThread:org.apache.tez.dag.app.rm.TaskSchedulerManager] |rm.TaskSchedulerManager|: Creating YARN TaskScheduler: org.apache.tez.dag.app.rm.YarnTaskSchedulerService 2019-04-16 12:37:26,621 [INFO] [ServiceThread:org.apache.tez.dag.app.rm.TaskSchedulerManager] |conf.Configuration|: resource-types.xml not found 2019-04-16 12:37:26,622 [INFO] [ServiceThread:org.apache.tez.dag.app.rm.TaskSchedulerManager] |resource.ResourceUtils|: Unable to find 'resource-types.xml'. 2019-04-16 12:37:26,652 [INFO] [ServiceThread:org.apache.tez.dag.app.rm.TaskSchedulerManager] |Configuration.deprecation|: io.bytes.per.checksum is deprecated. Instead, use dfs.bytes-per-checksum 2019-04-16 12:37:26,653 [INFO] [ServiceThread:org.apache.tez.dag.app.rm.TaskSchedulerManager] |Configuration.deprecation|: yarn.resourcemanager.system-metrics-publisher.enabled is deprecated. Instead, use yarn.system-metrics-publisher.enabled 2019-04-16 12:37:26,653 [INFO] [ServiceThread:org.apache.tez.dag.app.rm.TaskSchedulerManager] |rm.TaskSchedulerManager|: Creating TaskScheduler: Local TaskScheduler with clusterIdentifier=111101111 2019-04-16 12:37:26,667 [INFO] [ServiceThread:org.apache.tez.dag.app.rm.TaskSchedulerManager] |rm.YarnTaskSchedulerService|: YarnTaskScheduler initialized with configuration: maxRMHeartbeatInterval: 250, containerReuseEnabled: true, reuseRackLocal: true, reuseNonLocal: false, localitySchedulingDelay: 250, preemptionPercentage: 10, preemptionMaxWaitTime: 60000, numHeartbeatsBetweenPreemptions: 3, idleContainerMinTimeout: 10000, idleContainerMaxTimeout: 20000, sessionMinHeldContainers: 0 2019-04-16 12:37:26,676 [INFO] [ServiceThread:org.apache.tez.dag.app.rm.TaskSchedulerManager] |client.RMProxy|: Connecting to ResourceManager at ip-172-31-69-167.ec2.internal/172.31.69.167:8030 2019-04-16 12:37:26,792 [INFO] [main] |HistoryEventHandler.criticalEvents|: [HISTORY][DAG:N/A][Event:AM_STARTED]: appAttemptId=appattempt_1555415227039_0004_000001, startTime=1555418246791 2019-04-16 12:37:26,792 [INFO] [main] |app.DAGAppMaster|: In Session mode. Waiting for DAG over RPC 2019-04-16 12:37:26,830 [INFO] [AMRM Callback Handler Thread] |rm.YarnTaskSchedulerService|: App total resource memory: 32768 cpu: 8 taskAllocations: 0 2019-04-16 12:37:26,830 [INFO] [AMRM Callback Handler Thread] |rm.YarnTaskSchedulerService|: Allocated: Free: pendingRequests: 0 delayedContainers: 0 heartbeats: 1 lastPreemptionHeartbeat: 0 2019-04-16 12:37:26,833 [INFO] [Dispatcher thread {Central}] |node.PerSourceNodeTracker|: Num cluster nodes = 3 2019-04-16 12:37:27,314 [INFO] [IPC Server handler 1 on 38482] |app.DAGAppMaster|: Starting DAG submitted via RPC: Select T.Candidate.startTime, T.Candidat...T (Stage-1) 2019-04-16 12:37:27,316 [INFO] [IPC Server handler 1 on 38482] |app.DAGAppMaster|: Merging AM credentials into DAG credentials 2019-04-16 12:37:27,362 [INFO] [IPC Server handler 1 on 38482] |app.DAGAppMaster|: Generating DAG graphviz file, dagId=dag_1555415227039_0004_1, filePath=/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000001/dag_1555415227039_0004_1.dot 2019-04-16 12:37:27,367 [INFO] [IPC Server handler 1 on 38482] |common.TezUtilsInternal|: Redirecting log file based on addend: dag_1555415227039_0004_1 End of LogType:syslog *********************************************************************** Container: container_e06_1555415227039_0004_01_000001 on ip-172-31-83-216.ec2.internal_45454_1555418368508 LogAggregationType: AGGREGATED ========================================================================================================== LogType:syslog_dag_1555415227039_0004_1 LogLastModifiedTime:Tue Apr 16 12:39:28 +0000 2019 LogLength:26389 LogContents: 2019-04-16 12:37:27,367 [INFO] [IPC Server handler 1 on 38482] |app.DAGAppMaster|: Running DAG: Select T.Candidate.startTime, T.Candidat...T (Stage-1), callerContext={ context=HIVE, callerType=HIVE_QUERY_ID, callerId=hive_20190416123702_5ee0ce4b-526b-4828-b572-3bb9507e0f85 } 2019-04-16 12:37:27,486 [INFO] [IPC Server handler 1 on 38482] |HistoryEventHandler.criticalEvents|: [HISTORY][DAG:dag_1555415227039_0004_1][Event:DAG_SUBMITTED]: dagID=dag_1555415227039_0004_1, submitTime=1555418247315, queueName=default 2019-04-16 12:37:27,495 [INFO] [HistoryEventHandlingThread] |compress.CodecPool|: Got brand-new compressor [.deflate] 2019-04-16 12:37:27,534 [INFO] [IPC Server handler 1 on 38482] |impl.VertexImpl|: Default container context for vertex_1555415227039_0004_1_00 [Map 1]=LocalResources: [[ name=hive-exec-3.1.0.3.0.1.0-187-ffcc945cef1d45a4f8c86dab4e04eaea5bb4284dae0788c8b7f4df27141566d9.jar, value=resource { scheme: "hdfs" host: "ip-172-31-69-167.ec2.internal" port: 8020 file: "/user/hive/.hiveJars/hive-exec-3.1.0.3.0.1.0-187-ffcc945cef1d45a4f8c86dab4e04eaea5bb4284dae0788c8b7f4df27141566d9.jar" } size: 41297919 timestamp: 1554714629487 type: FILE visibility: PRIVATE],[ name=hive-hcatalog-core.jar, value=resource { scheme: "hdfs" host: "ip-172-31-69-167.ec2.internal" port: 8020 file: "/tmp/hive/hive/_tez_session_dir/b7360cc1-1e10-4d86-9994-5f1342034d5d-resources/hive-hcatalog-core.jar" } size: 269078 timestamp: 1555418242682 type: FILE visibility: PRIVATE],[ name=tez-conf.pb, value=resource { scheme: "hdfs" host: "ip-172-31-69-167.ec2.internal" port: 8020 file: "/tmp/hive/hive/_tez_session_dir/b7360cc1-1e10-4d86-9994-5f1342034d5d/.tez/application_1555415227039_0004/tez-conf.pb" } size: 125998 timestamp: 1555418242790 type: FILE visibility: APPLICATION],[ name=tezlib, value=resource { scheme: "hdfs" host: "ip-172-31-69-167.ec2.internal" port: 8020 file: "/hdp/apps/3.0.1.0-187/tez/tez.tar.gz" } size: 254013856 timestamp: 1554714421261 type: ARCHIVE visibility: PUBLIC],[ name=TimestampUDF-1.0-SNAPSHOT.jar, value=resource { scheme: "hdfs" host: "ip-172-31-69-167.ec2.internal" port: 8020 file: "/tmp/hive/hive/_tez_session_dir/b7360cc1-1e10-4d86-9994-5f1342034d5d-resources/TimestampUDF-1.0-SNAPSHOT.jar" } size: 3909 timestamp: 1555418242706 type: FILE visibility: PRIVATE],], environment: [[ SHELL=/bin/bash ],[ LD_LIBRARY_PATH=$PWD:/usr/hdp/3.0.1.0-187/hadoop/lib/native:/usr/hdp/3.0.1.0-187/hadoop/lib/native/Linux-amd64-64:$PWD:/usr/hdp/3.0.1.0-187/hadoop/lib/native:/usr/hdp/3.0.1.0-187/hadoop/lib/native/Linux-amd64-64:$PWD:/usr/hdp/3.0.1.0-187/hadoop/lib/native:/usr/hdp/3.0.1.0-187/hadoop/lib/native/Linux-amd64-64:$PWD:/usr/hdp/3.0.1.0-187/hadoop/lib/native:/usr/hdp/3.0.1.0-187/hadoop/lib/native/Linux-amd64-64:$HADOOP_COMMON_HOME/lib/native/ ],[ HADOOP_ROOT_LOGGER=INFO,CLA ],[ CLASSPATH=/usr/hdp/3.0.1.0-187/hadoop/lib/hadoop-lzo-0.6.0.3.0.1.0-187.jar:/etc/hadoop/conf/secure:$PWD:$PWD/*:$PWD/tezlib/*:$PWD/tezlib/lib/*: ],], credentials(token kinds): [tez.job,], javaOpts: -server -Djava.net.preferIPv4Stack=true -Dhdp.version=3.0.1.0-187 -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB -server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator -Dlog4j.configuration=tez-container-log4j.properties -Dyarn.app.container.log.dir= -Dtez.root.logger=INFO,CLA , vertex: vertex_1555415227039_0004_1_00 [Map 1], Default Resources= 2019-04-16 12:37:27,534 [INFO] [IPC Server handler 1 on 38482] |impl.VertexImpl|: Setting 1 additional inputs for vertexvertex_1555415227039_0004_1_00 [Map 1] 2019-04-16 12:37:27,534 [INFO] [IPC Server handler 1 on 38482] |impl.VertexImpl|: Setting 1 additional outputs for vertex vertex_1555415227039_0004_1_00 [Map 1] 2019-04-16 12:37:27,535 [INFO] [IPC Server handler 1 on 38482] |impl.VertexImpl|: Using ExecutionContext from Vertex for Vertex Map 1 2019-04-16 12:37:27,535 [INFO] [IPC Server handler 1 on 38482] |impl.VertexImpl|: Running vertex: vertex_1555415227039_0004_1_00 [Map 1] : TaskScheduler=0:TezYarn, ContainerLauncher=0:TezYarn, TaskCommunicator=0:TezYarn 2019-04-16 12:37:27,536 [INFO] [IPC Server handler 1 on 38482] |impl.DAGImpl|: Using DAG Scheduler: org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrder 2019-04-16 12:37:27,537 [INFO] [IPC Server handler 1 on 38482] |HistoryEventHandler.criticalEvents|: [HISTORY][DAG:dag_1555415227039_0004_1][Event:DAG_INITIALIZED]: dagID=dag_1555415227039_0004_1, initTime=1555418247537 2019-04-16 12:37:27,537 [INFO] [IPC Server handler 1 on 38482] |impl.DAGImpl|: dag_1555415227039_0004_1 transitioned from NEW to INITED due to event DAG_INIT 2019-04-16 12:37:27,539 [INFO] [Dispatcher thread {Central}] |impl.DAGImpl|: Added additional resources : [[]] to classpath 2019-04-16 12:37:27,541 [INFO] [Dispatcher thread {Central}] |HistoryEventHandler.criticalEvents|: [HISTORY][DAG:dag_1555415227039_0004_1][Event:DAG_STARTED]: dagID=dag_1555415227039_0004_1, startTime=1555418247538 2019-04-16 12:37:27,541 [INFO] [Dispatcher thread {Central}] |impl.DAGImpl|: dag_1555415227039_0004_1 transitioned from INITED to RUNNING due to event DAG_START 2019-04-16 12:37:27,541 [INFO] [Dispatcher thread {Central}] |impl.VertexImpl|: Root Inputs exist for Vertex: Map 1 : {idtable={InputName=idtable}, {Descriptor=ClassName=org.apache.tez.mapreduce.input.MRInputLegacy, hasPayload=true}, {ControllerDescriptor=ClassName=org.apache.hadoop.hive.ql.exec.tez.HiveSplitGenerator, hasPayload=false}} 2019-04-16 12:37:27,542 [INFO] [Dispatcher thread {Central}] |impl.VertexImpl|: Starting root input initializer for input: idtable, with class: [org.apache.hadoop.hive.ql.exec.tez.HiveSplitGenerator] 2019-04-16 12:37:27,542 [INFO] [Dispatcher thread {Central}] |impl.VertexImpl|: Setting vertexManager to RootInputVertexManager for vertex_1555415227039_0004_1_00 [Map 1] 2019-04-16 12:37:27,558 [INFO] [Dispatcher thread {Central}] |impl.VertexImpl|: Num tasks is -1. Expecting VertexManager/InputInitializers/1-1 split to set #tasks for the vertex vertex_1555415227039_0004_1_00 [Map 1] 2019-04-16 12:37:27,558 [INFO] [Dispatcher thread {Central}] |impl.VertexImpl|: Vertex will initialize from input initializer. vertex_1555415227039_0004_1_00 [Map 1] 2019-04-16 12:37:27,559 [INFO] [Dispatcher thread {Central}] |impl.VertexImpl|: Starting 1 inputInitializers for vertex vertex_1555415227039_0004_1_00 [Map 1] 2019-04-16 12:37:27,575 [INFO] [Dispatcher thread {Central}] |Configuration.deprecation|: mapred.committer.job.setup.cleanup.needed is deprecated. Instead, use mapreduce.job.committer.setup.cleanup.needed 2019-04-16 12:37:27,721 [INFO] [Dispatcher thread {Central}] |conf.HiveConf|: Found configuration file null 2019-04-16 12:37:27,724 [INFO] [Dispatcher thread {Central}] |exec.Utilities|: PLAN PATH = hdfs://ip-172-31-69-167.ec2.internal:8020/tmp/hive/hive/4adc7ed2-01cc-481b-b2aa-bbe41551ac08/hive_2019-04-16_12-37-02_586_6615575765504587239-2/hive/_tez_scratch_dir/8187bcb8-4a61-4ade-8498-5577705993a2/map.xml 2019-04-16 12:37:27,731 [INFO] [Dispatcher thread {Central}] |exec.SerializationUtilities|: Deserializing MapWork using kryo 2019-04-16 12:37:27,868 [INFO] [Dispatcher thread {Central}] |exec.Utilities|: Deserialized plan (via RPC) - name: Map 1 size: 3.14KB 2019-04-16 12:37:27,869 [INFO] [Dispatcher thread {Central}] |tez.HiveSplitGenerator|: SplitGenerator using llap affinitized locations: false 2019-04-16 12:37:27,869 [INFO] [Dispatcher thread {Central}] |tez.HiveSplitGenerator|: SplitLocationProvider: org.apache.hadoop.hive.ql.exec.tez.Utils$1@2aac303d 2019-04-16 12:37:27,874 [INFO] [InputInitializer {Map 1} #0] |dag.RootInputInitializerManager|: Starting InputInitializer for Input: idtable on vertex vertex_1555415227039_0004_1_00 [Map 1] 2019-04-16 12:37:27,875 [INFO] [InputInitializer {Map 1} #0] |tez.HiveSplitGenerator|: GenerateConsistentSplitsInHive=true 2019-04-16 12:37:27,881 [INFO] [Dispatcher thread {Central}] |impl.VertexImpl|: vertex_1555415227039_0004_1_00 [Map 1] transitioned from NEW to INITIALIZING due to event V_INIT 2019-04-16 12:37:27,882 [INFO] [InputInitializer {Map 1} #0] |tez.HiveSplitGenerator|: The preferred split size is 16777216 2019-04-16 12:37:27,883 [INFO] [InputInitializer {Map 1} #0] |exec.Utilities|: PLAN PATH = hdfs://ip-172-31-69-167.ec2.internal:8020/tmp/hive/hive/4adc7ed2-01cc-481b-b2aa-bbe41551ac08/hive_2019-04-16_12-37-02_586_6615575765504587239-2/hive/_tez_scratch_dir/8187bcb8-4a61-4ade-8498-5577705993a2/map.xml 2019-04-16 12:37:27,886 [INFO] [IPC Server handler 0 on 38482] |counters.Limits|: Counter limits initialized with parameters: GROUP_NAME_MAX=256, MAX_GROUPS=3000, COUNTER_NAME_MAX=64, MAX_COUNTERS=10000 2019-04-16 12:37:27,889 [INFO] [InputInitializer {Map 1} #0] |exec.Utilities|: Processing alias idtable 2019-04-16 12:37:27,889 [INFO] [InputInitializer {Map 1} #0] |exec.Utilities|: Adding 1 inputs; the first input is hdfs://ip-172-31-69-167.ec2.internal:8020/warehouse/tablespace/managed/hive/idtable 2019-04-16 12:37:27,892 [INFO] [InputInitializer {Map 1} #0] |io.HiveInputFormat|: hive.io.file.readcolumn.ids = 2019-04-16 12:37:27,892 [INFO] [InputInitializer {Map 1} #0] |io.HiveInputFormat|: hive.io.file.readcolumn.names = 2019-04-16 12:37:27,892 [INFO] [InputInitializer {Map 1} #0] |io.HiveInputFormat|: Generating splits for dirs: hdfs://ip-172-31-69-167.ec2.internal:8020/warehouse/tablespace/managed/hive/idtable 2019-04-16 12:37:27,911 [INFO] [InputInitializer {Map 1} #0] |orc.OrcInputFormat|: ORC pushdown predicate: null 2019-04-16 12:37:27,911 [INFO] [InputInitializer {Map 1} #0] |Configuration.deprecation|: mapred.input.dir is deprecated. Instead, use mapreduce.input.fileinputformat.inputdir 2019-04-16 12:37:28,017 [INFO] [ORC_GET_SPLITS #1] |impl.OrcCodecPool|: Got brand-new codec ZLIB 2019-04-16 12:37:28,042 [INFO] [InputInitializer {Map 1} #0] |orc.OrcInputFormat|: FooterCacheHitRatio: 0/1 2019-04-16 12:37:28,042 [INFO] [InputInitializer {Map 1} #0] |io.HiveInputFormat|: number of splits 1 2019-04-16 12:37:28,043 [INFO] [InputInitializer {Map 1} #0] |tez.HiveSplitGenerator|: Number of input splits: 1. 8 available slots, 1.7 waves. Input format is: org.apache.hadoop.hive.ql.io.HiveInputFormat 2019-04-16 12:37:28,044 [INFO] [InputInitializer {Map 1} #0] |exec.Utilities|: Processing alias idtable 2019-04-16 12:37:28,044 [INFO] [InputInitializer {Map 1} #0] |exec.Utilities|: Adding 1 inputs; the first input is hdfs://ip-172-31-69-167.ec2.internal:8020/warehouse/tablespace/managed/hive/idtable 2019-04-16 12:37:28,044 [INFO] [InputInitializer {Map 1} #0] |exec.Utilities|: PLAN PATH = hdfs://ip-172-31-69-167.ec2.internal:8020/tmp/hive/hive/4adc7ed2-01cc-481b-b2aa-bbe41551ac08/hive_2019-04-16_12-37-02_586_6615575765504587239-2/hive/_tez_scratch_dir/8187bcb8-4a61-4ade-8498-5577705993a2/map.xml 2019-04-16 12:37:28,045 [INFO] [InputInitializer {Map 1} #0] |exec.SerializationUtilities|: Deserializing MapWork using kryo 2019-04-16 12:37:28,052 [INFO] [InputInitializer {Map 1} #0] |exec.Utilities|: Deserialized plan (via RPC) - name: Map 1 size: 3.14KB 2019-04-16 12:37:28,053 [INFO] [InputInitializer {Map 1} #0] |tez.SplitGrouper|: # Src groups for split generation: 2 2019-04-16 12:37:28,054 [INFO] [InputInitializer {Map 1} #0] |tez.SplitGrouper|: Estimated number of tasks: 13 for bucket 1 2019-04-16 12:37:28,056 [INFO] [InputInitializer {Map 1} #0] |grouper.TezSplitGrouper|: Grouping splits in Tez 2019-04-16 12:37:28,056 [INFO] [InputInitializer {Map 1} #0] |grouper.TezSplitGrouper|: Desired splits: 1 too large. Desired splitLength: 1 Min splitLength: 16777216 New desired splits: 1 Final desired splits: 1 All splits have localhost: false Total length: 14 Original splits: 1 2019-04-16 12:37:28,056 [INFO] [InputInitializer {Map 1} #0] |grouper.TezSplitGrouper|: Using original number of splits: 1 desired splits: 1 2019-04-16 12:37:28,058 [INFO] [InputInitializer {Map 1} #0] |tez.SplitGrouper|: Original split count is 1 grouped split count is 1, for bucket: 1 2019-04-16 12:37:28,058 [INFO] [InputInitializer {Map 1} #0] |tez.HiveSplitGenerator|: Number of split groups: 1 2019-04-16 12:37:28,069 [INFO] [InputInitializer {Map 1} #0] |dag.RootInputInitializerManager|: Succeeded InputInitializer for Input: idtable on vertex vertex_1555415227039_0004_1_00 [Map 1] 2019-04-16 12:37:28,072 [INFO] [App Shared Pool - #0] |impl.VertexImpl|: Cannot init vertex: vertex_1555415227039_0004_1_00 [Map 1] numTasks: -1 numUnitializedEdges: 0 numInitializedInputs: 1 initWaitsForRootInitializers: true 2019-04-16 12:37:28,072 [INFO] [App Shared Pool - #0] |impl.VertexImpl|: Got updated RootInputsSpecs: {idtable=forAllWorkUnits=true, update=[1]} 2019-04-16 12:37:28,080 [INFO] [App Shared Pool - #0] |impl.VertexImpl|: Vertex vertex_1555415227039_0004_1_00 [Map 1] parallelism set to 1 2019-04-16 12:37:28,081 [INFO] [App Shared Pool - #0] |impl.VertexImpl|: Cannot init vertex: vertex_1555415227039_0004_1_00 [Map 1] numTasks: 1 numUnitializedEdges: 0 numInitializedInputs: 1 initWaitsForRootInitializers: true 2019-04-16 12:37:28,091 [INFO] [Dispatcher thread {Central}] |impl.VertexImpl|: Setting up committers for vertex vertex_1555415227039_0004_1_00 [Map 1], numAdditionalOutputs=1 2019-04-16 12:37:28,092 [INFO] [Dispatcher thread {Central}] |HistoryEventHandler.criticalEvents|: [HISTORY][DAG:dag_1555415227039_0004_1][Event:VERTEX_INITIALIZED]: vertexName=Map 1, vertexId=vertex_1555415227039_0004_1_00, initRequestedTime=1555418247541, initedTime=1555418248091, numTasks=1, processorName=org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor, additionalInputsCount=1, initGeneratedEventsCount=1, servicePluginInfo=ServicePluginInfo {containerLauncherName=TezYarn, taskSchedulerName=TezYarn, taskCommunicatorName=TezYarn, containerLauncherClassName=org.apache.tez.dag.app.launcher.TezContainerLauncherImpl, taskSchedulerClassName=org.apache.tez.dag.app.rm.YarnTaskSchedulerService, taskCommunicatorClassName=org.apache.tez.dag.app.TezTaskCommunicatorImpl } 2019-04-16 12:37:28,092 [INFO] [Dispatcher thread {Central}] |impl.VertexImpl|: Triggering start event for vertex: vertex_1555415227039_0004_1_00 [Map 1] with distanceFromRoot: 0 2019-04-16 12:37:28,092 [INFO] [Dispatcher thread {Central}] |impl.VertexImpl|: vertex_1555415227039_0004_1_00 [Map 1] transitioned from INITIALIZING to INITED due to event V_INPUT_DATA_INFORMATION 2019-04-16 12:37:28,095 [INFO] [Dispatcher thread {Central}] |HistoryEventHandler.criticalEvents|: [HISTORY][DAG:dag_1555415227039_0004_1][Event:VERTEX_STARTED]: vertexId=vertex_1555415227039_0004_1_00, startRequestedTime=1555418247884, startedTime=1555418248092 2019-04-16 12:37:28,096 [INFO] [Dispatcher thread {Central}] |HistoryEventHandler.criticalEvents|: [HISTORY][DAG:dag_1555415227039_0004_1][Event:VERTEX_CONFIGURE_DONE]: vertexId=vertex_1555415227039_0004_1_00, reconfigureDoneTime=1555418248096, numTasks=1, vertexLocationHint=org.apache.tez.dag.api.VertexLocationHint@d20690f0, edgeManagersCount=0, rootInputSpecUpdateCount=1, setParallelismCalledFlag=true 2019-04-16 12:37:28,096 [INFO] [Dispatcher thread {Central}] |impl.VertexImpl|: vertex_1555415227039_0004_1_00 [Map 1] transitioned from INITED to RUNNING due to event V_START 2019-04-16 12:37:28,097 [INFO] [App Shared Pool - #0] |impl.RootInputVertexManager|: All source tasks completed. Ramping up 1 remaining tasks for vertex: Map 1 2019-04-16 12:37:28,097 [INFO] [App Shared Pool - #0] |impl.RootInputVertexManager|: Scheduling 1 tasks for vertex: Map 1 with totalTasks: 1. 0 source tasks completed out of 0. MinSourceTaskCompletedFraction: 1.0 min: 0.0 max: 0.0 2019-04-16 12:37:28,097 [INFO] [App Shared Pool - #0] |impl.VertexImpl|: Routing pending task events for vertex: vertex_1555415227039_0004_1_00 [Map 1] 2019-04-16 12:37:28,101 [INFO] [Dispatcher thread {Central}] |HistoryEventHandler.criticalEvents|: [HISTORY][DAG:dag_1555415227039_0004_1][Event:TASK_STARTED]: vertexName=Map 1, taskId=task_1555415227039_0004_1_00_000000, scheduledTime=1555418248100, launchTime=1555418248100 2019-04-16 12:37:28,198 [INFO] [TaskSchedulerEventHandlerThread] |rm.YarnTaskSchedulerService|: Allocation request for task: attempt_1555415227039_0004_1_00_000000_0 with request: Capability[]Priority[2]AllocationRequestId[0]ExecutionTypeRequest[{Execution Type: GUARANTEED, Enforce Execution Type: false}]Resource Profile[null] host: ip-172-31-93-195.ec2.internal rack: null 2019-04-16 12:37:28,615 [INFO] [DelayedContainerManager] |rm.YarnTaskSchedulerService|: Assigning container to task: containerId=container_e06_1555415227039_0004_01_000002, task=attempt_1555415227039_0004_1_00_000000_0, containerHost=ip-172-31-88-76.ec2.internal:45454, containerPriority= 2, containerResources=, localityMatchType=NodeLocal, matchedLocation=ip-172-31-88-76.ec2.internal, honorLocalityFlags=true, reusedContainer=false, delayedContainers=0 2019-04-16 12:37:28,631 [INFO] [TaskSchedulerAppCallbackExecutor #0] |node.PerSourceNodeTracker|: Adding new node ip-172-31-88-76.ec2.internal:45454 to nodeTracker 0 2019-04-16 12:37:28,685 [INFO] [ContainerLauncher #0] |launcher.TezContainerLauncherImpl|: Launching container_e06_1555415227039_0004_01_000002 2019-04-16 12:37:29,157 [INFO] [ContainerLauncher #0] |HistoryEventHandler.criticalEvents|: [HISTORY][DAG:N/A][Event:CONTAINER_LAUNCHED]: containerId=container_e06_1555415227039_0004_01_000002, launchTime=1555418249157 2019-04-16 12:37:31,575 [INFO] [Socket Reader #1 for port 45483] |ipc.Server|: Auth successful for application_1555415227039_0004 (auth:SIMPLE) 2019-04-16 12:37:31,597 [INFO] [Dispatcher thread {Central}] |impl.TaskAttemptImpl|: TaskAttempt: [attempt_1555415227039_0004_1_00_000000_0] submitted. Is using containerId: [container_e06_1555415227039_0004_01_000002] on NM: [ip-172-31-88-76.ec2.internal:45454] 2019-04-16 12:37:31,598 [INFO] [Dispatcher thread {Central}] |HistoryEventHandler.criticalEvents|: [HISTORY][DAG:dag_1555415227039_0004_1][Event:TASK_ATTEMPT_STARTED]: vertexName=Map 1, taskAttemptId=attempt_1555415227039_0004_1_00_000000_0, startTime=1555418251597, containerId=container_e06_1555415227039_0004_01_000002, nodeId=ip-172-31-88-76.ec2.internal:45454 2019-04-16 12:37:31,732 [INFO] [IPC Server handler 1 on 45483] |impl.TaskImpl|: TaskAttempt:attempt_1555415227039_0004_1_00_000000_0 sent events: (0-1). 2019-04-16 12:37:39,450 [INFO] [AMRM Callback Handler Thread] |rm.YarnTaskSchedulerService|: Allocated: Free: pendingRequests: 0 delayedContainers: 0 heartbeats: 51 lastPreemptionHeartbeat: 50 2019-04-16 12:37:39,712 [INFO] [Dispatcher thread {Central}] |HistoryEventHandler.criticalEvents|: [HISTORY][DAG:dag_1555415227039_0004_1][Event:TASK_ATTEMPT_FINISHED]: vertexName=Map 1, taskAttemptId=attempt_1555415227039_0004_1_00_000000_0, creationTime=1555418248113, allocationTime=1555418248654, startTime=1555418251597, finishTime=1555418259710, timeTaken=8113, status=SUCCEEDED, diagnostics= 2019-04-16 12:37:39,714 [INFO] [Dispatcher thread {Central}] |impl.TaskImpl|: Task succeeded with attempt attempt_1555415227039_0004_1_00_000000_0 2019-04-16 12:37:39,718 [INFO] [Dispatcher thread {Central}] |HistoryEventHandler.criticalEvents|: [HISTORY][DAG:dag_1555415227039_0004_1][Event:TASK_FINISHED]: vertexName=Map 1, taskId=task_1555415227039_0004_1_00_000000, startTime=1555418251597, finishTime=1555418259714, timeTaken=8117, status=SUCCEEDED, successfulAttemptID=attempt_1555415227039_0004_1_00_000000_0, diagnostics=, counters=Counters: 29, File System Counters, HDFS_BYTES_READ=712, HDFS_BYTES_WRITTEN=154, HDFS_READ_OPS=5, HDFS_WRITE_OPS=2, HDFS_OP_CREATE=1, HDFS_OP_GET_FILE_STATUS=3, HDFS_OP_OPEN=2, HDFS_OP_RENAME=1, org.apache.tez.common.counters.TaskCounter, GC_TIME_MILLIS=79, TASK_DURATION_MILLIS=8113, CPU_MILLISECONDS=3480, PHYSICAL_MEMORY_BYTES=253755392, VIRTUAL_MEMORY_BYTES=5422989312, COMMITTED_HEAP_BYTES=253755392, INPUT_RECORDS_PROCESSED=1, INPUT_SPLIT_LENGTH_BYTES=276, OUTPUT_RECORDS=0, HIVE, CREATED_FILES=1, DESERIALIZE_ERRORS=0, RECORDS_IN_Map_1=1, RECORDS_OUT_0=1, RECORDS_OUT_INTERMEDIATE_Map_1=0, RECORDS_OUT_OPERATOR_FS_4=1, RECORDS_OUT_OPERATOR_MAP_0=0, RECORDS_OUT_OPERATOR_SEL_3=1, RECORDS_OUT_OPERATOR_TS_0=1, TaskCounter_Map_1_INPUT_idtable, INPUT_RECORDS_PROCESSED=1, INPUT_SPLIT_LENGTH_BYTES=276, TaskCounter_Map_1_OUTPUT_out_Map_1, OUTPUT_RECORDS=0 2019-04-16 12:37:39,722 [INFO] [Dispatcher thread {Central}] |impl.VertexImpl|: Task Completion: vertex_1555415227039_0004_1_00 [Map 1], tasks=1, failed=0, killed=0, success=1, completed=1, commits=0, err=null 2019-04-16 12:37:39,722 [INFO] [Dispatcher thread {Central}] |impl.VertexImpl|: All tasks have succeeded, vertex:vertex_1555415227039_0004_1_00 [Map 1] 2019-04-16 12:37:39,730 [INFO] [Dispatcher thread {Central}] |HistoryEventHandler.criticalEvents|: [HISTORY][DAG:dag_1555415227039_0004_1][Event:VERTEX_FINISHED]: vertexName=Map 1, vertexId=vertex_1555415227039_0004_1_00, initRequestedTime=1555418247541, initedTime=1555418248091, startRequestedTime=1555418247884, startedTime=1555418248092, finishTime=1555418259722, timeTaken=11630, status=SUCCEEDED, diagnostics=, counters=Counters: 33, File System Counters, HDFS_BYTES_READ=712, HDFS_BYTES_WRITTEN=154, HDFS_READ_OPS=5, HDFS_WRITE_OPS=2, HDFS_OP_CREATE=1, HDFS_OP_GET_FILE_STATUS=3, HDFS_OP_OPEN=2, HDFS_OP_RENAME=1, org.apache.tez.common.counters.TaskCounter, GC_TIME_MILLIS=79, TASK_DURATION_MILLIS=8113, CPU_MILLISECONDS=3480, PHYSICAL_MEMORY_BYTES=253755392, VIRTUAL_MEMORY_BYTES=5422989312, COMMITTED_HEAP_BYTES=253755392, INPUT_RECORDS_PROCESSED=1, INPUT_SPLIT_LENGTH_BYTES=276, OUTPUT_RECORDS=0, HIVE, CREATED_FILES=1, DESERIALIZE_ERRORS=0, RECORDS_IN_Map_1=1, RECORDS_OUT_0=1, RECORDS_OUT_INTERMEDIATE_Map_1=0, RECORDS_OUT_OPERATOR_FS_4=1, RECORDS_OUT_OPERATOR_MAP_0=0, RECORDS_OUT_OPERATOR_SEL_3=1, RECORDS_OUT_OPERATOR_TS_0=1, TaskCounter_Map_1_INPUT_idtable, INPUT_RECORDS_PROCESSED=1, INPUT_SPLIT_LENGTH_BYTES=276, TaskCounter_Map_1_OUTPUT_out_Map_1, OUTPUT_RECORDS=0, org.apache.hadoop.hive.ql.exec.tez.HiveInputCounters, GROUPED_INPUT_SPLITS_Map_1=1, INPUT_DIRECTORIES_Map_1=1, INPUT_FILES_Map_1=1, RAW_INPUT_SPLITS_Map_1=1, vertexStats=firstTaskStartTime=1555418251597, firstTasksToStart=[ task_1555415227039_0004_1_00_000000 ], lastTaskFinishTime=1555418259710, lastTasksToFinish=[ task_1555415227039_0004_1_00_000000 ], minTaskDuration=8113, maxTaskDuration=8113, avgTaskDuration=8113.0, numSuccessfulTasks=1, shortestDurationTasks=[ task_1555415227039_0004_1_00_000000 ], longestDurationTasks=[ task_1555415227039_0004_1_00_000000 ], vertexTaskStats={numFailedTaskAttempts=0, numKilledTaskAttempts=0, numCompletedTasks=1, numSucceededTasks=1, numKilledTasks=0, numFailedTasks=0}, servicePluginInfo=ServicePluginInfo {containerLauncherName=TezYarn, taskSchedulerName=TezYarn, taskCommunicatorName=TezYarn, containerLauncherClassName=org.apache.tez.dag.app.launcher.TezContainerLauncherImpl, taskSchedulerClassName=org.apache.tez.dag.app.rm.YarnTaskSchedulerService, taskCommunicatorClassName=org.apache.tez.dag.app.TezTaskCommunicatorImpl } 2019-04-16 12:37:39,730 [INFO] [Dispatcher thread {Central}] |impl.VertexImpl|: vertex_1555415227039_0004_1_00 [Map 1] transitioned from RUNNING to SUCCEEDED due to event V_TASK_COMPLETED 2019-04-16 12:37:39,730 [INFO] [Dispatcher thread {Central}] |impl.DAGImpl|: Vertex vertex_1555415227039_0004_1_00 [Map 1] completed., numCompletedVertices=1, numSuccessfulVertices=1, numFailedVertices=0, numKilledVertices=0, numVertices=1 2019-04-16 12:37:39,730 [INFO] [Dispatcher thread {Central}] |impl.DAGImpl|: Checking vertices for DAG completion, numCompletedVertices=1, numSuccessfulVertices=1, numFailedVertices=0, numKilledVertices=0, numVertices=1, commitInProgress=0, terminationCause=null 2019-04-16 12:37:39,730 [INFO] [Dispatcher thread {Central}] |impl.DAGImpl|: No exclusive output committers for vertex: vertex_1555415227039_0004_1_00 [Map 1] 2019-04-16 12:37:39,740 [INFO] [Dispatcher thread {Central}] |recovery.RecoveryService|: DAG completed, dagId=dag_1555415227039_0004_1, queueSize=0 2019-04-16 12:37:39,751 [INFO] [Dispatcher thread {Central}] |HistoryEventHandler.criticalEvents|: [HISTORY][DAG:dag_1555415227039_0004_1][Event:DAG_FINISHED]: dagId=dag_1555415227039_0004_1, startTime=1555418247538, finishTime=1555418259730, timeTaken=12192, status=SUCCEEDED, diagnostics=, counters=Counters: 37, org.apache.tez.common.counters.DAGCounter, NUM_SUCCEEDED_TASKS=1, TOTAL_LAUNCHED_TASKS=1, AM_CPU_MILLISECONDS=2350, AM_GC_TIME_MILLIS=26, File System Counters, HDFS_BYTES_READ=712, HDFS_BYTES_WRITTEN=154, HDFS_READ_OPS=5, HDFS_WRITE_OPS=2, HDFS_OP_CREATE=1, HDFS_OP_GET_FILE_STATUS=3, HDFS_OP_OPEN=2, HDFS_OP_RENAME=1, org.apache.tez.common.counters.TaskCounter, GC_TIME_MILLIS=79, TASK_DURATION_MILLIS=8113, CPU_MILLISECONDS=3480, PHYSICAL_MEMORY_BYTES=253755392, VIRTUAL_MEMORY_BYTES=5422989312, COMMITTED_HEAP_BYTES=253755392, INPUT_RECORDS_PROCESSED=1, INPUT_SPLIT_LENGTH_BYTES=276, OUTPUT_RECORDS=0, HIVE, CREATED_FILES=1, DESERIALIZE_ERRORS=0, RECORDS_IN_Map_1=1, RECORDS_OUT_0=1, RECORDS_OUT_INTERMEDIATE_Map_1=0, RECORDS_OUT_OPERATOR_FS_4=1, RECORDS_OUT_OPERATOR_MAP_0=0, RECORDS_OUT_OPERATOR_SEL_3=1, RECORDS_OUT_OPERATOR_TS_0=1, TaskCounter_Map_1_INPUT_idtable, INPUT_RECORDS_PROCESSED=1, INPUT_SPLIT_LENGTH_BYTES=276, TaskCounter_Map_1_OUTPUT_out_Map_1, OUTPUT_RECORDS=0, org.apache.hadoop.hive.ql.exec.tez.HiveInputCounters, GROUPED_INPUT_SPLITS_Map_1=1, INPUT_DIRECTORIES_Map_1=1, INPUT_FILES_Map_1=1, RAW_INPUT_SPLITS_Map_1=1 2019-04-16 12:37:39,751 [INFO] [Dispatcher thread {Central}] |impl.DAGImpl|: DAG: dag_1555415227039_0004_1 finished with state: SUCCEEDED 2019-04-16 12:37:39,751 [INFO] [Dispatcher thread {Central}] |impl.DAGImpl|: dag_1555415227039_0004_1 transitioned from RUNNING to SUCCEEDED due to event DAG_VERTEX_COMPLETED 2019-04-16 12:37:39,752 [INFO] [Dispatcher thread {Central}] |app.DAGAppMaster|: DAG completed, dagId=dag_1555415227039_0004_1, dagState=SUCCEEDED 2019-04-16 12:37:39,752 [INFO] [Dispatcher thread {Central}] |common.TezUtilsInternal|: Redirecting log file based on addend: dag_1555415227039_0004_1_post End of LogType:syslog_dag_1555415227039_0004_1 ************************************************************************************************ Container: container_e06_1555415227039_0004_01_000001 on ip-172-31-83-216.ec2.internal_45454_1555418368508 LogAggregationType: AGGREGATED ========================================================================================================== LogType:syslog_dag_1555415227039_0004_1_post LogLastModifiedTime:Tue Apr 16 12:39:28 +0000 2019 LogLength:9043 LogContents: 2019-04-16 12:37:39,757 [INFO] [Dispatcher thread {Central}] |app.DAGAppMaster|: Central Dispatcher queue size after DAG completion, before cleanup: 0 2019-04-16 12:37:39,757 [INFO] [Dispatcher thread {Central}] |app.DAGAppMaster|: Waiting for next DAG to be submitted. 2019-04-16 12:37:39,758 [INFO] [Dispatcher thread {Central}] |app.DAGAppMaster|: Cleaning up DAG: name=Select T.Candidate.startTime, T.Candidat...T (Stage-1), with id=dag_1555415227039_0004_1 2019-04-16 12:37:39,759 [INFO] [Dispatcher thread {Central}] |container.AMContainerMap|: Cleaned up completed containers on dagComplete. Removed=0, Remaining=1 2019-04-16 12:37:39,759 [INFO] [Dispatcher thread {Central}] |app.DAGAppMaster|: Completed cleanup for DAG: name=Select T.Candidate.startTime, T.Candidat...T (Stage-1), with id=dag_1555415227039_0004_1 2019-04-16 12:37:39,761 [INFO] [HistoryEventHandlingThread] |compress.CodecPool|: Got brand-new compressor [.deflate] 2019-04-16 12:37:52,053 [INFO] [AMRM Callback Handler Thread] |rm.YarnTaskSchedulerService|: Allocated: Free: pendingRequests: 0 delayedContainers: 1 heartbeats: 101 lastPreemptionHeartbeat: 100 2019-04-16 12:37:54,223 [INFO] [DelayedContainerManager] |rm.YarnTaskSchedulerService|: No taskRequests. Container's idle timeout delay expired or is new. Releasing container, containerId=container_e06_1555415227039_0004_01_000002, containerExpiryTime=1555418274099, idleTimeout=10000, taskRequestsCount=0, heldContainers=1, delayedContainers=0, isNew=false 2019-04-16 12:37:54,225 [INFO] [Dispatcher thread {Central}] |HistoryEventHandler.criticalEvents|: [HISTORY][DAG:dag_1555415227039_0004_1][Event:CONTAINER_STOPPED]: containerId=container_e06_1555415227039_0004_01_000002, stoppedTime=1555418274225, exitStatus=0 2019-04-16 12:37:54,226 [INFO] [ContainerLauncher #1] |launcher.TezContainerLauncherImpl|: Stopping container_e06_1555415227039_0004_01_000002 2019-04-16 12:37:54,327 [INFO] [Dispatcher thread {Central}] |container.AMContainerImpl|: Container container_e06_1555415227039_0004_01_000002 exited with diagnostics set to Container failed, exitCode=-105. [2019-04-16 12:37:54.239]Container killed by the ApplicationMaster. [2019-04-16 12:37:54.264]Container killed on request. Exit code is 143 [2019-04-16 12:37:54.270]Container exited with a non-zero exit code 143. 2019-04-16 12:38:04,655 [INFO] [AMRM Callback Handler Thread] |rm.YarnTaskSchedulerService|: Allocated: Free: pendingRequests: 0 delayedContainers: 0 heartbeats: 151 lastPreemptionHeartbeat: 150 2019-04-16 12:38:17,253 [INFO] [AMRM Callback Handler Thread] |rm.YarnTaskSchedulerService|: Allocated: Free: pendingRequests: 0 delayedContainers: 0 heartbeats: 201 lastPreemptionHeartbeat: 200 2019-04-16 12:38:29,848 [INFO] [AMRM Callback Handler Thread] |rm.YarnTaskSchedulerService|: Allocated: Free: pendingRequests: 0 delayedContainers: 0 heartbeats: 251 lastPreemptionHeartbeat: 250 2019-04-16 12:38:42,443 [INFO] [AMRM Callback Handler Thread] |rm.YarnTaskSchedulerService|: Allocated: Free: pendingRequests: 0 delayedContainers: 0 heartbeats: 301 lastPreemptionHeartbeat: 300 2019-04-16 12:38:55,035 [INFO] [AMRM Callback Handler Thread] |rm.YarnTaskSchedulerService|: Allocated: Free: pendingRequests: 0 delayedContainers: 0 heartbeats: 351 lastPreemptionHeartbeat: 350 2019-04-16 12:39:07,634 [INFO] [AMRM Callback Handler Thread] |rm.YarnTaskSchedulerService|: Allocated: Free: pendingRequests: 0 delayedContainers: 0 heartbeats: 401 lastPreemptionHeartbeat: 400 2019-04-16 12:39:20,223 [INFO] [AMRM Callback Handler Thread] |rm.YarnTaskSchedulerService|: Allocated: Free: pendingRequests: 0 delayedContainers: 0 heartbeats: 451 lastPreemptionHeartbeat: 450 2019-04-16 12:39:21,960 [INFO] [IPC Server handler 0 on 38482] |client.DAGClientHandler|: Received message to shutdown AM from hive (auth:SIMPLE) at 172.31.69.167 2019-04-16 12:39:21,960 [INFO] [IPC Server handler 0 on 38482] |rm.TaskSchedulerManager|: TaskScheduler notified that it should unregister from RM 2019-04-16 12:39:21,960 [INFO] [IPC Server handler 0 on 38482] |app.DAGAppMaster|: No current running DAG, shutting down the AM 2019-04-16 12:39:21,960 [INFO] [IPC Server handler 0 on 38482] |app.DAGAppMaster|: DAGAppMasterShutdownHandler invoked 2019-04-16 12:39:21,960 [INFO] [IPC Server handler 0 on 38482] |app.DAGAppMaster|: Handling DAGAppMaster shutdown 2019-04-16 12:39:21,961 [INFO] [AMShutdownThread] |app.DAGAppMaster|: Sleeping for 5000 ms before shutting down 2019-04-16 12:39:26,961 [INFO] [AMShutdownThread] |app.DAGAppMaster|: Calling stop for all the services 2019-04-16 12:39:26,961 [INFO] [AMShutdownThread] |rm.YarnTaskSchedulerService|: Initiating stop of YarnTaskScheduler 2019-04-16 12:39:26,961 [INFO] [AMShutdownThread] |rm.YarnTaskSchedulerService|: Releasing held containers 2019-04-16 12:39:26,961 [INFO] [AMShutdownThread] |rm.YarnTaskSchedulerService|: Removing all pending taskRequests 2019-04-16 12:39:26,962 [INFO] [AMShutdownThread] |history.HistoryEventHandler|: Stopping HistoryEventHandler 2019-04-16 12:39:26,962 [INFO] [AMShutdownThread] |recovery.RecoveryService|: Stopping RecoveryService 2019-04-16 12:39:26,962 [INFO] [AMShutdownThread] |recovery.RecoveryService|: Handle the remaining events in queue, queue size=0 2019-04-16 12:39:26,962 [INFO] [RecoveryEventHandlingThread] |recovery.RecoveryService|: EventQueue take interrupted. Returning 2019-04-16 12:39:26,962 [INFO] [AMShutdownThread] |recovery.RecoveryService|: Closing Summary Stream 2019-04-16 12:39:26,974 [WARN] [AMShutdownThread] |recovery.RecoveryService|: Ignoring error while closing summary stream. The recovery directory at hdfs://ip-172-31-69-167.ec2.internal:8020/tmp/hive/hive/_tez_session_dir/b7360cc1-1e10-4d86-9994-5f1342034d5d/.tez/application_1555415227039_0004/recovery/1 has already been deleted externally 2019-04-16 12:39:26,975 [INFO] [AMShutdownThread] |proto.ProtoHistoryLoggingService|: Stopping ProtoHistoryLoggingService, eventQueueBacklog=0 2019-04-16 12:39:27,009 [INFO] [AMShutdownThread] |proto.ProtoHistoryLoggingService|: Stopped ProtoHistoryLoggingService 2019-04-16 12:39:27,009 [INFO] [DelayedContainerManager] |rm.YarnTaskSchedulerService|: AllocatedContainerManager Thread interrupted 2019-04-16 12:39:27,010 [INFO] [AMShutdownThread] |rm.YarnTaskSchedulerService|: Unregistering application from RM, exitStatus=SUCCEEDED, exitMessage=Session stats:submittedDAGs=1, successfulDAGs=1, failedDAGs=0, killedDAGs=0 , trackingURL= 2019-04-16 12:39:27,015 [INFO] [AMShutdownThread] |impl.AMRMClientImpl|: Waiting for application to be successfully unregistered. 2019-04-16 12:39:27,116 [INFO] [AMShutdownThread] |rm.YarnTaskSchedulerService|: Successfully unregistered application from RM 2019-04-16 12:39:27,117 [INFO] [AMShutdownThread] |rm.TaskSchedulerManager|: Shutting down AppCallbackExecutor 2019-04-16 12:39:27,121 [INFO] [AMShutdownThread] |handler.ContextHandler|: Stopped o.e.j.w.WebAppContext@1aa1bf6e{/,null,UNAVAILABLE}{/} 2019-04-16 12:39:27,124 [INFO] [AMShutdownThread] |server.AbstractConnector|: Stopped ServerConnector@3be5a72a{HTTP/1.1,[http/1.1]}{0.0.0.0:0} 2019-04-16 12:39:27,124 [INFO] [AMShutdownThread] |handler.ContextHandler|: Stopped o.e.j.s.ServletContextHandler@3b5f737{/static,jar:file:/hadoop/yarn/local/filecache/10/tez.tar.gz/lib/hadoop-yarn-common-3.1.1.3.0.1.0-187.jar!/webapps/static,UNAVAILABLE} 2019-04-16 12:39:27,125 [INFO] [AMShutdownThread] |ipc.Server|: Stopping server on 45483 2019-04-16 12:39:27,125 [INFO] [AMShutdownThread] |ipc.Server|: Stopping server on 38482 2019-04-16 12:39:27,125 [INFO] [IPC Server Responder] |ipc.Server|: Stopping IPC Server Responder 2019-04-16 12:39:27,125 [INFO] [IPC Server listener on 45483] |ipc.Server|: Stopping IPC Server listener on 45483 2019-04-16 12:39:27,126 [INFO] [IPC Server listener on 38482] |ipc.Server|: Stopping IPC Server listener on 38482 2019-04-16 12:39:27,126 [INFO] [IPC Server Responder] |ipc.Server|: Stopping IPC Server Responder 2019-04-16 12:39:27,128 [INFO] [shutdown-hook-0] |app.DAGAppMaster|: DAGAppMasterShutdownHook invoked 2019-04-16 12:39:27,128 [INFO] [shutdown-hook-0] |app.DAGAppMaster|: The shutdown handler is still running, waiting for it to complete 2019-04-16 12:39:27,132 [WARN] [AMShutdownThread] |app.DAGAppMaster|: Failed to delete tez scratch data dir, path=hdfs://ip-172-31-69-167.ec2.internal:8020/tmp/hive/hive/_tez_session_dir/b7360cc1-1e10-4d86-9994-5f1342034d5d/.tez/application_1555415227039_0004 2019-04-16 12:39:27,132 [INFO] [AMShutdownThread] |app.DAGAppMaster|: Exiting DAGAppMaster..GoodBye! 2019-04-16 12:39:27,132 [INFO] [shutdown-hook-0] |app.DAGAppMaster|: The shutdown handler has completed End of LogType:syslog_dag_1555415227039_0004_1_post ***************************************************************************************************** Container: container_e06_1555415227039_0004_01_000002 on ip-172-31-88-76.ec2.internal_45454_1555418368531 LogAggregationType: AGGREGATED ========================================================================================================= LogType:directory.info LogLastModifiedTime:Tue Apr 16 12:39:28 +0000 2019 LogLength:23097 LogContents: ls -l: total 20 -rw-r--r-- 1 yarn hadoop 112 Apr 16 12:37 container_tokens -rwx------ 1 yarn hadoop 652 Apr 16 12:37 default_container_executor_session.sh -rwx------ 1 yarn hadoop 707 Apr 16 12:37 default_container_executor.sh lrwxrwxrwx 1 yarn hadoop 143 Apr 16 12:37 hive-exec-3.1.0.3.0.1.0-187-ffcc945cef1d45a4f8c86dab4e04eaea5bb4284dae0788c8b7f4df27141566d9.jar -> /hadoop/yarn/local/usercache/hive/filecache/10/hive-exec-3.1.0.3.0.1.0-187-ffcc945cef1d45a4f8c86dab4e04eaea5bb4284dae0788c8b7f4df27141566d9.jar lrwxrwxrwx 1 yarn hadoop 69 Apr 16 12:37 hive-hcatalog-core.jar -> /hadoop/yarn/local/usercache/hive/filecache/15/hive-hcatalog-core.jar -rwx------ 1 yarn hadoop 5415 Apr 16 12:37 launch_container.sh lrwxrwxrwx 1 yarn hadoop 98 Apr 16 12:37 tez-conf.pb -> /hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/filecache/10/tez-conf.pb lrwxrwxrwx 1 yarn hadoop 42 Apr 16 12:37 tezlib -> /hadoop/yarn/local/filecache/10/tez.tar.gz lrwxrwxrwx 1 yarn hadoop 76 Apr 16 12:37 TimestampUDF-1.0-SNAPSHOT.jar -> /hadoop/yarn/local/usercache/hive/filecache/16/TimestampUDF-1.0-SNAPSHOT.jar drwx--x--- 2 yarn hadoop 6 Apr 16 12:37 tmp find -L . -maxdepth 5 -ls: 360714345 4 drwx--x--- 3 yarn hadoop 4096 Apr 16 12:37 . 369110627 0 drwx--x--- 2 yarn hadoop 6 Apr 16 12:37 ./tmp 360714347 4 -rw-r--r-- 1 yarn hadoop 112 Apr 16 12:37 ./container_tokens 360714348 4 -rw-r--r-- 1 yarn hadoop 12 Apr 16 12:37 ./.container_tokens.crc 360714349 8 -rwx------ 1 yarn hadoop 5415 Apr 16 12:37 ./launch_container.sh 360714357 4 -rw-r--r-- 1 yarn hadoop 52 Apr 16 12:37 ./.launch_container.sh.crc 360714358 4 -rwx------ 1 yarn hadoop 652 Apr 16 12:37 ./default_container_executor_session.sh 360714359 4 -rw-r--r-- 1 yarn hadoop 16 Apr 16 12:37 ./.default_container_executor_session.sh.crc 360714360 4 -rwx------ 1 yarn hadoop 707 Apr 16 12:37 ./default_container_executor.sh 360714361 4 -rw-r--r-- 1 yarn hadoop 16 Apr 16 12:37 ./.default_container_executor.sh.crc 339746207 4 -r-x------ 1 yarn hadoop 3909 Apr 16 12:37 ./TimestampUDF-1.0-SNAPSHOT.jar 373294253 40332 -r-x------ 1 yarn hadoop 41297919 Apr 8 09:10 ./hive-exec-3.1.0.3.0.1.0-187-ffcc945cef1d45a4f8c86dab4e04eaea5bb4284dae0788c8b7f4df27141566d9.jar 331356806 264 -r-x------ 1 yarn hadoop 269078 Apr 16 12:37 ./hive-hcatalog-core.jar 348129458 124 -r-x------ 1 yarn hadoop 125998 Apr 16 12:37 ./tez-conf.pb 364939014 4 drwxr-xr-x 3 yarn hadoop 4096 Apr 8 09:06 ./tezlib 364939015 4 -r-xr-xr-x 1 yarn hadoop 1465 Sep 19 2018 ./tezlib/LICENSE-BSD-3clause 364939016 20 -r-xr-xr-x 1 yarn hadoop 17257 Sep 19 2018 ./tezlib/LICENSE 364939017 4 -r-xr-xr-x 1 yarn hadoop 1509 Sep 19 2018 ./tezlib/NOTICE 364939018 20 -r-xr-xr-x 1 yarn hadoop 16792 Sep 19 2018 ./tezlib/LICENSE-CDDLv1.0 364939019 4 -r-xr-xr-x 1 yarn hadoop 1045 Sep 19 2018 ./tezlib/LICENSE-MIT 364939020 8 -r-xr-xr-x 1 yarn hadoop 4128 Sep 19 2018 ./tezlib/LICENSE-SIL_OpenFontLicense-v1.1 364939021 36 -r-xr-xr-x 1 yarn hadoop 36261 Sep 19 2018 ./tezlib/LICENSE-CDDLv1.1-GPLv2_withCPE 364939022 12 -r-xr-xr-x 1 yarn hadoop 8899 Sep 19 2018 ./tezlib/hadoop-shim-0.9.1.3.0.1.0-187.jar 364939023 1020 -r-xr-xr-x 1 yarn hadoop 1041820 Sep 19 2018 ./tezlib/tez-api-0.9.1.3.0.1.0-187.jar 364939024 84 -r-xr-xr-x 1 yarn hadoop 85946 Sep 19 2018 ./tezlib/tez-common-0.9.1.3.0.1.0-187.jar 364939025 196 -r-xr-xr-x 1 yarn hadoop 199548 Sep 19 2018 ./tezlib/tez-runtime-internals-0.9.1.3.0.1.0-187.jar 364939028 752 -r-xr-xr-x 1 yarn hadoop 768635 Sep 19 2018 ./tezlib/tez-runtime-library-0.9.1.3.0.1.0-187.jar 364939029 292 -r-xr-xr-x 1 yarn hadoop 296385 Sep 19 2018 ./tezlib/tez-mapreduce-0.9.1.3.0.1.0-187.jar 364939030 56 -r-xr-xr-x 1 yarn hadoop 56876 Sep 19 2018 ./tezlib/tez-examples-0.9.1.3.0.1.0-187.jar 364939031 1380 -r-xr-xr-x 1 yarn hadoop 1409888 Sep 19 2018 ./tezlib/tez-dag-0.9.1.3.0.1.0-187.jar 364939032 156 -r-xr-xr-x 1 yarn hadoop 158850 Sep 19 2018 ./tezlib/tez-tests-0.9.1.3.0.1.0-187.jar 364939033 108 -r-xr-xr-x 1 yarn hadoop 107597 Sep 19 2018 ./tezlib/tez-ext-service-tests-0.9.1.3.0.1.0-187.jar 364939034 108616 -r-xr-xr-x 1 yarn hadoop 111220116 Sep 19 2018 ./tezlib/tez-ui-0.9.1.3.0.1.0-187.war 364939035 76 -r-xr-xr-x 1 yarn hadoop 75760 Sep 19 2018 ./tezlib/tez-protobuf-history-plugin-0.9.1.3.0.1.0-187.jar 364939036 28 -r-xr-xr-x 1 yarn hadoop 28273 Sep 19 2018 ./tezlib/tez-yarn-timeline-history-0.9.1.3.0.1.0-187.jar 364939037 8 -r-xr-xr-x 1 yarn hadoop 7839 Sep 19 2018 ./tezlib/tez-yarn-timeline-history-with-acls-0.9.1.3.0.1.0-187.jar 364939038 12 -r-xr-xr-x 1 yarn hadoop 11563 Sep 19 2018 ./tezlib/tez-yarn-timeline-cache-plugin-0.9.1.3.0.1.0-187.jar 364939039 24 -r-xr-xr-x 1 yarn hadoop 23691 Sep 19 2018 ./tezlib/tez-yarn-timeline-history-with-fs-0.9.1.3.0.1.0-187.jar 364939040 80 -r-xr-xr-x 1 yarn hadoop 79219 Sep 19 2018 ./tezlib/tez-history-parser-0.9.1.3.0.1.0-187.jar 364939041 80 -r-xr-xr-x 1 yarn hadoop 79101 Sep 19 2018 ./tezlib/tez-job-analyzer-0.9.1.3.0.1.0-187.jar 364939042 16 -r-xr-xr-x 1 yarn hadoop 15352 Sep 19 2018 ./tezlib/tez-javadoc-tools-0.9.1.3.0.1.0-187.jar 364939043 8 -r-xr-xr-x 1 yarn hadoop 6445 Sep 19 2018 ./tezlib/hadoop-shim-2.8-0.9.1.3.0.1.0-187.jar 402653962 12 drwxr-xr-x 3 yarn hadoop 8192 Apr 8 09:06 ./tezlib/lib 402653963 32 -r-xr-xr-x 1 yarn hadoop 32119 Jun 22 2018 ./tezlib/lib/slf4j-api-1.7.10.jar 402653964 3028 -r-xr-xr-x 1 yarn hadoop 3098999 Sep 19 2018 ./tezlib/lib/hadoop-yarn-api-3.1.1.3.0.1.0-187.jar 402653965 280 -r-xr-xr-x 1 yarn hadoop 284220 Jun 22 2018 ./tezlib/lib/commons-lang-2.6.jar 402653966 1612 -r-xr-xr-x 1 yarn hadoop 1648200 Sep 19 2018 ./tezlib/lib/guava-11.0.2.jar 402653969 36 -r-xr-xr-x 1 yarn hadoop 33031 Sep 19 2018 ./tezlib/lib/jsr305-3.0.0.jar 402653970 64 -r-xr-xr-x 1 yarn hadoop 62050 Jun 22 2018 ./tezlib/lib/commons-logging-1.1.3.jar 402653971 100 -r-xr-xr-x 1 yarn hadoop 102244 Jun 22 2018 ./tezlib/lib/jaxb-api-2.2.11.jar 402653980 524 -r-xr-xr-x 1 yarn hadoop 533455 Sep 19 2018 ./tezlib/lib/protobuf-java-2.5.0.jar 402653981 68 -r-xr-xr-x 1 yarn hadoop 66983 Jun 22 2018 ./tezlib/lib/jackson-annotations-2.9.5.jar 402653982 3952 -r-xr-xr-x 1 yarn hadoop 4043890 Sep 19 2018 ./tezlib/lib/hadoop-common-3.1.1.3.0.1.0-187.jar 402653983 60 -r-xr-xr-x 1 yarn hadoop 59895 Sep 19 2018 ./tezlib/lib/hadoop-annotations-3.1.1.3.0.1.0-187.jar 402653984 44 -r-xr-xr-x 1 yarn hadoop 41123 Jun 22 2018 ./tezlib/lib/commons-cli-1.2.jar 402653985 1564 -r-xr-xr-x 1 yarn hadoop 1599627 Jun 22 2018 ./tezlib/lib/commons-math3-3.1.1.jar 402653986 720 -r-xr-xr-x 1 yarn hadoop 736658 Jun 22 2018 ./tezlib/lib/httpclient-4.5.2.jar 402653987 320 -r-xr-xr-x 1 yarn hadoop 326724 Jun 22 2018 ./tezlib/lib/httpcore-4.4.4.jar 402653988 60 -r-xr-xr-x 1 yarn hadoop 58160 Jun 22 2018 ./tezlib/lib/commons-codec-1.4.jar 402653989 184 -r-xr-xr-x 1 yarn hadoop 185140 Jun 22 2018 ./tezlib/lib/commons-io-2.4.jar 402653990 304 -r-xr-xr-x 1 yarn hadoop 307410 Jun 22 2018 ./tezlib/lib/commons-net-3.6.jar 402653991 576 -r-xr-xr-x 1 yarn hadoop 588337 Jun 22 2018 ./tezlib/lib/commons-collections-3.2.2.jar 402653992 96 -r-xr-xr-x 1 yarn hadoop 95806 Jun 22 2018 ./tezlib/lib/javax.servlet-api-3.1.0.jar 402653993 120 -r-xr-xr-x 1 yarn hadoop 119119 Jun 22 2018 ./tezlib/lib/jetty-servlet-9.3.19.v20170502.jar 402653994 96 -r-xr-xr-x 1 yarn hadoop 95838 Jun 22 2018 ./tezlib/lib/jetty-security-9.3.19.v20170502.jar 402653995 112 -r-xr-xr-x 1 yarn hadoop 113395 Jun 22 2018 ./tezlib/lib/jetty-webapp-9.3.19.v20170502.jar 402653996 52 -r-xr-xr-x 1 yarn hadoop 50633 Jun 22 2018 ./tezlib/lib/jetty-xml-9.3.19.v20170502.jar 402653997 128 -r-xr-xr-x 1 yarn hadoop 128719 Jun 22 2018 ./tezlib/lib/jersey-servlet-1.19.jar 402653998 164 -r-xr-xr-x 1 yarn hadoop 165345 Jun 22 2018 ./tezlib/lib/jersey-json-1.19.jar 402653999 80 -r-xr-xr-x 1 yarn hadoop 81743 Jun 22 2018 ./tezlib/lib/jettison-1.3.4.jar 402654000 872 -r-xr-xr-x 1 yarn hadoop 890168 Jun 22 2018 ./tezlib/lib/jaxb-impl-2.2.3-1.jar 402654001 224 -r-xr-xr-x 1 yarn hadoop 228286 Jun 22 2018 ./tezlib/lib/jackson-core-asl-1.9.2.jar 402654002 748 -r-xr-xr-x 1 yarn hadoop 765648 Jun 22 2018 ./tezlib/lib/jackson-mapper-asl-1.9.2.jar 402654003 20 -r-xr-xr-x 1 yarn hadoop 18323 Jun 22 2018 ./tezlib/lib/jackson-jaxrs-1.9.2.jar 402654004 28 -r-xr-xr-x 1 yarn hadoop 27075 Jun 22 2018 ./tezlib/lib/jackson-xc-1.9.2.jar 402654005 480 -r-xr-xr-x 1 yarn hadoop 489884 Jun 22 2018 ./tezlib/lib/log4j-1.2.17.jar 402654006 244 -r-xr-xr-x 1 yarn hadoop 246174 Jun 22 2018 ./tezlib/lib/commons-beanutils-1.9.3.jar 402654007 604 -r-xr-xr-x 1 yarn hadoop 616888 Jun 22 2018 ./tezlib/lib/commons-configuration2-2.1.1.jar 402654008 428 -r-xr-xr-x 1 yarn hadoop 434678 Jun 22 2018 ./tezlib/lib/commons-lang3-3.4.jar 402654009 12 -r-xr-xr-x 1 yarn hadoop 8866 Jun 22 2018 ./tezlib/lib/slf4j-log4j12-1.7.10.jar 402654010 428 -r-xr-xr-x 1 yarn hadoop 436303 Jun 22 2018 ./tezlib/lib/avro-1.7.7.jar 402654011 32 -r-xr-xr-x 1 yarn hadoop 29555 Jun 22 2018 ./tezlib/lib/paranamer-2.3.jar 402654012 1224 -r-xr-xr-x 1 yarn hadoop 1251514 Jun 22 2018 ./tezlib/lib/snappy-java-1.0.5.jar 402654013 236 -r-xr-xr-x 1 yarn hadoop 241367 Jun 22 2018 ./tezlib/lib/commons-compress-1.4.1.jar 402654014 96 -r-xr-xr-x 1 yarn hadoop 94672 Jun 22 2018 ./tezlib/lib/xz-1.0.jar 402654015 128 -r-xr-xr-x 1 yarn hadoop 128414 Sep 19 2018 ./tezlib/lib/re2j-1.1.jar 402653320 188 -r-xr-xr-x 1 yarn hadoop 190432 Sep 19 2018 ./tezlib/lib/gson-2.2.4.jar 402653321 136 -r-xr-xr-x 1 yarn hadoop 138358 Sep 19 2018 ./tezlib/lib/hadoop-auth-3.1.1.3.0.1.0-187.jar 402653322 296 -r-xr-xr-x 1 yarn hadoop 299508 Jun 22 2018 ./tezlib/lib/nimbus-jose-jwt-4.41.1.jar 402653323 8 -r-xr-xr-x 1 yarn hadoop 4722 Jun 22 2018 ./tezlib/lib/jcip-annotations-1.0-1.jar 402653324 120 -r-xr-xr-x 1 yarn hadoop 120316 Jun 22 2018 ./tezlib/lib/json-smart-2.3.jar 402653325 32 -r-xr-xr-x 1 yarn hadoop 30035 Jun 22 2018 ./tezlib/lib/accessors-smart-1.2.jar 402653326 56 -r-xr-xr-x 1 yarn hadoop 53297 Jun 22 2018 ./tezlib/lib/asm-5.0.4.jar 402653376 788 -r-xr-xr-x 1 yarn hadoop 805053 Sep 19 2018 ./tezlib/lib/zookeeper-3.4.6.3.0.1.0-187.jar 402653377 1300 -r-xr-xr-x 1 yarn hadoop 1330394 Jun 22 2018 ./tezlib/lib/netty-3.10.5.Final.jar 402653378 200 -r-xr-xr-x 1 yarn hadoop 201953 Jun 22 2018 ./tezlib/lib/curator-framework-2.12.0.jar 402653379 2368 -r-xr-xr-x 1 yarn hadoop 2423138 Jun 22 2018 ./tezlib/lib/curator-client-2.12.0.jar 402653380 20 -r-xr-xr-x 1 yarn hadoop 20409 Jun 22 2018 ./tezlib/lib/kerb-simplekdc-1.0.1.jar 402653381 112 -r-xr-xr-x 1 yarn hadoop 113017 Jun 22 2018 ./tezlib/lib/kerb-client-1.0.1.jar 402653382 32 -r-xr-xr-x 1 yarn hadoop 30674 Jun 22 2018 ./tezlib/lib/kerby-config-1.0.1.jar 402653383 224 -r-xr-xr-x 1 yarn hadoop 226672 Jun 22 2018 ./tezlib/lib/kerb-core-1.0.1.jar 402653384 200 -r-xr-xr-x 1 yarn hadoop 204650 Jun 22 2018 ./tezlib/lib/kerby-pkix-1.0.1.jar 402653385 100 -r-xr-xr-x 1 yarn hadoop 102174 Jun 22 2018 ./tezlib/lib/kerby-asn1-1.0.1.jar 402653386 40 -r-xr-xr-x 1 yarn hadoop 40554 Jun 22 2018 ./tezlib/lib/kerby-util-1.0.1.jar 402653387 64 -r-xr-xr-x 1 yarn hadoop 65464 Jun 22 2018 ./tezlib/lib/kerb-common-1.0.1.jar 402653388 116 -r-xr-xr-x 1 yarn hadoop 116120 Jun 22 2018 ./tezlib/lib/kerb-crypto-1.0.1.jar 402653389 36 -r-xr-xr-x 1 yarn hadoop 36708 Jun 22 2018 ./tezlib/lib/kerb-util-1.0.1.jar 402653390 20 -r-xr-xr-x 1 yarn hadoop 18763 Jun 22 2018 ./tezlib/lib/token-provider-1.0.1.jar 402653391 80 -r-xr-xr-x 1 yarn hadoop 80980 Jun 22 2018 ./tezlib/lib/kerb-admin-1.0.1.jar 402653392 84 -r-xr-xr-x 1 yarn hadoop 82756 Jun 22 2018 ./tezlib/lib/kerb-server-1.0.1.jar 402653393 20 -r-xr-xr-x 1 yarn hadoop 20046 Jun 22 2018 ./tezlib/lib/kerb-identity-1.0.1.jar 402653394 32 -r-xr-xr-x 1 yarn hadoop 29134 Jun 22 2018 ./tezlib/lib/kerby-xdr-1.0.1.jar 402653395 280 -r-xr-xr-x 1 yarn hadoop 283598 Jun 22 2018 ./tezlib/lib/curator-recipes-2.12.0.jar 402653396 1468 -r-xr-xr-x 1 yarn hadoop 1502280 Jun 22 2018 ./tezlib/lib/htrace-core4-4.1.0-incubating.jar 402653397 88 -r-xr-xr-x 1 yarn hadoop 87325 Jun 22 2018 ./tezlib/lib/jline-0.9.94.jar 402653398 1312 -r-xr-xr-x 1 yarn hadoop 1342410 Jun 22 2018 ./tezlib/lib/jackson-databind-2.9.5.jar 402653399 316 -r-xr-xr-x 1 yarn hadoop 321590 Jun 22 2018 ./tezlib/lib/jackson-core-2.9.5.jar 402653400 160 -r-xr-xr-x 1 yarn hadoop 161867 Jun 22 2018 ./tezlib/lib/stax2-api-3.1.4.jar 402653401 504 -r-xr-xr-x 1 yarn hadoop 512742 Jun 22 2018 ./tezlib/lib/woodstox-core-5.0.3.jar 402653402 448 -r-xr-xr-x 1 yarn hadoop 456857 Sep 19 2018 ./tezlib/lib/hadoop-aws-3.1.1.3.0.1.0-187.jar 402653403 84200 -r-xr-xr-x 1 yarn hadoop 86220098 Jun 22 2018 ./tezlib/lib/aws-java-sdk-bundle-1.11.271.jar 402653404 544 -r-xr-xr-x 1 yarn hadoop 556241 Jun 22 2018 ./tezlib/lib/netty-codec-http-4.1.17.Final.jar 402653405 308 -r-xr-xr-x 1 yarn hadoop 313647 Jun 22 2018 ./tezlib/lib/netty-codec-4.1.17.Final.jar 402653406 444 -r-xr-xr-x 1 yarn hadoop 452023 Jun 22 2018 ./tezlib/lib/netty-transport-4.1.17.Final.jar 402653407 268 -r-xr-xr-x 1 yarn hadoop 270353 Jun 22 2018 ./tezlib/lib/netty-buffer-4.1.17.Final.jar 402653408 556 -r-xr-xr-x 1 yarn hadoop 565466 Jun 22 2018 ./tezlib/lib/netty-common-4.1.17.Final.jar 402653409 32 -r-xr-xr-x 1 yarn hadoop 32235 Jun 22 2018 ./tezlib/lib/netty-resolver-4.1.17.Final.jar 402653410 368 -r-xr-xr-x 1 yarn hadoop 373661 Jun 22 2018 ./tezlib/lib/netty-handler-4.1.17.Final.jar 402653411 464 -r-xr-xr-x 1 yarn hadoop 472196 Sep 19 2018 ./tezlib/lib/hadoop-azure-3.1.1.3.0.1.0-187.jar 402653412 796 -r-xr-xr-x 1 yarn hadoop 812977 Jun 22 2018 ./tezlib/lib/azure-storage-7.0.0.jar 402653413 12 -r-xr-xr-x 1 yarn hadoop 10288 Jun 22 2018 ./tezlib/lib/azure-keyvault-core-1.0.0.jar 402653414 656 -r-xr-xr-x 1 yarn hadoop 668235 Sep 19 2018 ./tezlib/lib/guice-4.0.jar 402653415 4 -r-xr-xr-x 1 yarn hadoop 2497 Jun 22 2018 ./tezlib/lib/javax.inject-1.jar 402653416 8 -r-xr-xr-x 1 yarn hadoop 4467 Jun 22 2018 ./tezlib/lib/aopalliance-1.0.jar 402653417 40 -r-xr-xr-x 1 yarn hadoop 38707 Jun 22 2018 ./tezlib/lib/jetty-util-ajax-9.3.19.v20170502.jar 402653418 448 -r-xr-xr-x 1 yarn hadoop 456916 Jun 22 2018 ./tezlib/lib/jetty-util-9.3.22.v20171030.jar 402653419 428 -r-xr-xr-x 1 yarn hadoop 437222 Aug 21 2018 ./tezlib/lib/wildfly-openssl-1.0.4.Final.jar 402653420 68 -r-xr-xr-x 1 yarn hadoop 66891 Sep 19 2018 ./tezlib/lib/hadoop-azure-datalake-3.1.1.3.0.1.0-187.jar 402653421 92 -r-xr-xr-x 1 yarn hadoop 92683 Jun 22 2018 ./tezlib/lib/azure-data-lake-store-sdk-2.2.7.jar 402653422 3420 -r-xr-xr-x 1 yarn hadoop 3500453 Sep 19 2018 ./tezlib/lib/gcs-connector-1.9.0.3.0.1.0-187-shaded.jar 402653423 28 -r-xr-xr-x 1 yarn hadoop 26514 Jun 22 2018 ./tezlib/lib/stax-api-1.0.1.jar 402653424 2760 -r-xr-xr-x 1 yarn hadoop 2825438 Sep 19 2018 ./tezlib/lib/hadoop-yarn-common-3.1.1.3.0.1.0-187.jar 402653425 428 -r-xr-xr-x 1 yarn hadoop 436689 Jun 22 2018 ./tezlib/lib/jersey-core-1.19.jar 402653426 48 -r-xr-xr-x 1 yarn hadoop 46367 Jun 22 2018 ./tezlib/lib/jsr311-api-1.1.1.jar 402653427 132 -r-xr-xr-x 1 yarn hadoop 134021 Jun 22 2018 ./tezlib/lib/jersey-client-1.19.jar 402653428 76 -r-xr-xr-x 1 yarn hadoop 76983 Sep 19 2018 ./tezlib/lib/guice-servlet-4.0.jar 402653429 16 -r-xr-xr-x 1 yarn hadoop 16151 Jun 22 2018 ./tezlib/lib/jersey-guice-1.19.jar 402653430 32 -r-xr-xr-x 1 yarn hadoop 32618 Jun 22 2018 ./tezlib/lib/jackson-module-jaxb-annotations-2.9.5.jar 402653431 16 -r-xr-xr-x 1 yarn hadoop 15864 Jun 22 2018 ./tezlib/lib/jackson-jaxrs-json-provider-2.9.5.jar 402653432 32 -r-xr-xr-x 1 yarn hadoop 32373 Jun 22 2018 ./tezlib/lib/jackson-jaxrs-base-2.9.5.jar 402653433 296 -r-xr-xr-x 1 yarn hadoop 300895 Sep 19 2018 ./tezlib/lib/hadoop-yarn-client-3.1.1.3.0.1.0-187.jar 402653434 736 -r-xr-xr-x 1 yarn hadoop 751238 Jun 22 2018 ./tezlib/lib/commons-collections4-4.1.jar 402653435 512 -r-xr-xr-x 1 yarn hadoop 520532 Jun 22 2018 ./tezlib/lib/jetty-server-9.3.22.v20171030.jar 402653436 148 -r-xr-xr-x 1 yarn hadoop 150378 Jun 22 2018 ./tezlib/lib/jetty-http-9.3.22.v20171030.jar 402653437 120 -r-xr-xr-x 1 yarn hadoop 122664 Jun 22 2018 ./tezlib/lib/jetty-io-9.3.22.v20171030.jar 402653438 24 -r-xr-xr-x 1 yarn hadoop 24239 Jun 22 2018 ./tezlib/lib/commons-daemon-1.0.13.jar 402653439 1024 -r-xr-xr-x 1 yarn hadoop 1045744 Jun 22 2018 ./tezlib/lib/leveldbjni-all-1.8.jar 402655136 4872 -r-xr-xr-x 1 yarn hadoop 4986914 Sep 19 2018 ./tezlib/lib/hadoop-hdfs-client-3.1.1.3.0.1.0-187.jar 402655137 324 -r-xr-xr-x 1 yarn hadoop 331034 Jun 22 2018 ./tezlib/lib/okhttp-2.7.5.jar 402655138 68 -r-xr-xr-x 1 yarn hadoop 65928 Jun 22 2018 ./tezlib/lib/okio-1.6.0.jar 402655139 124 -r-xr-xr-x 1 yarn hadoop 124846 Jun 22 2018 ./tezlib/lib/RoaringBitmap-0.4.9.jar 402655140 728 -r-xr-xr-x 1 yarn hadoop 745264 Jun 22 2018 ./tezlib/lib/async-http-client-1.9.40.jar 402655141 1616 -r-xr-xr-x 1 yarn hadoop 1654668 Sep 19 2018 ./tezlib/lib/hadoop-mapreduce-client-core-3.1.1.3.0.1.0-187.jar 402655142 788 -r-xr-xr-x 1 yarn hadoop 803565 Sep 19 2018 ./tezlib/lib/hadoop-mapreduce-client-common-3.1.1.3.0.1.0-187.jar 402655143 80 -r-xr-xr-x 1 yarn hadoop 80462 Sep 19 2018 ./tezlib/lib/hadoop-yarn-server-web-proxy-3.1.1.3.0.1.0-187.jar 402655144 1308 -r-xr-xr-x 1 yarn hadoop 1336861 Sep 19 2018 ./tezlib/lib/hadoop-yarn-server-common-3.1.1.3.0.1.0-187.jar 402655145 224 -r-xr-xr-x 1 yarn hadoop 225710 Sep 19 2018 ./tezlib/lib/hadoop-yarn-registry-3.1.1.3.0.1.0-187.jar 402655146 304 -r-xr-xr-x 1 yarn hadoop 307637 Jun 22 2018 ./tezlib/lib/dnsjava-2.1.7.jar 402655147 56 -r-xr-xr-x 1 yarn hadoop 55236 Jun 22 2018 ./tezlib/lib/geronimo-jcache_1.0_spec-1.0-alpha-1.jar 402655148 1688 -r-xr-xr-x 1 yarn hadoop 1726527 Jun 22 2018 ./tezlib/lib/ehcache-3.3.1.jar 402655149 132 -r-xr-xr-x 1 yarn hadoop 134308 Jun 22 2018 ./tezlib/lib/HikariCP-java7-2.4.12.jar 402655150 776 -r-xr-xr-x 1 yarn hadoop 792442 Jun 22 2018 ./tezlib/lib/mssql-jdbc-6.2.1.jre7.jar 402655151 104 -r-xr-xr-x 1 yarn hadoop 105112 Jun 22 2018 ./tezlib/lib/servlet-api-2.5.jar 402655152 112 -r-xr-xr-x 1 yarn hadoop 111908 Jun 22 2018 ./tezlib/lib/metrics-core-3.1.0.jar 402655153 288 -r-xr-xr-x 1 yarn hadoop 294856 Sep 19 2018 ./tezlib/lib/hadoop-yarn-server-applicationhistoryservice-3.1.1.3.0.1.0-187.jar 402655154 28 -r-xr-xr-x 1 yarn hadoop 28569 Jun 22 2018 ./tezlib/lib/objenesis-1.0.jar 402655155 380 -r-xr-xr-x 1 yarn hadoop 387689 Jun 22 2018 ./tezlib/lib/fst-2.50.jar 402655156 60 -r-xr-xr-x 1 yarn hadoop 58487 Jun 22 2018 ./tezlib/lib/java-util-1.9.0.jar 402655157 76 -r-xr-xr-x 1 yarn hadoop 75232 Jun 22 2018 ./tezlib/lib/json-io-2.5.1.jar 402655158 92 -r-xr-xr-x 1 yarn hadoop 93140 Sep 19 2018 ./tezlib/lib/hadoop-yarn-server-timeline-pluginstorage-3.1.1.3.0.1.0-187.jar 406850327 4 drwxr-xr-x 3 yarn hadoop 4096 Sep 19 2018 ./tezlib/lib/native 406850328 3324 -r-xr-xr-x 2 yarn hadoop 3402313 Sep 19 2018 ./tezlib/lib/native/libnativetask.so 411056614 0 drwxr-xr-x 2 yarn hadoop 94 Sep 19 2018 ./tezlib/lib/native/examples 411056615 964 -r-xr-xr-x 1 yarn hadoop 983078 Sep 19 2018 ./tezlib/lib/native/examples/wordcount-nopipe 411056616 920 -r-xr-xr-x 1 yarn hadoop 938380 Sep 19 2018 ./tezlib/lib/native/examples/wordcount-simple 411056617 924 -r-xr-xr-x 1 yarn hadoop 943052 Sep 19 2018 ./tezlib/lib/native/examples/wordcount-part 411056618 904 -r-xr-xr-x 1 yarn hadoop 924089 Sep 19 2018 ./tezlib/lib/native/examples/pipes-sort 406850329 468 -r-xr-xr-x 1 yarn hadoop 476136 Sep 19 2018 ./tezlib/lib/native/libhadooputils.a 406850330 8396 -r-xr-xr-x 1 yarn hadoop 8594282 Sep 19 2018 ./tezlib/lib/native/libnativetask.a 406850331 24 -r-xr-xr-x 3 yarn hadoop 23800 Jun 10 2014 ./tezlib/lib/native/libsnappy.so.1.1.4 406850332 280 -r-xr-xr-x 2 yarn hadoop 286676 Sep 19 2018 ./tezlib/lib/native/libhdfs.so.0.0.0 406850328 3324 -r-xr-xr-x 2 yarn hadoop 3402313 Sep 19 2018 ./tezlib/lib/native/libnativetask.so.1.0.0 406850333 948 -r-xr-xr-x 2 yarn hadoop 968926 Sep 19 2018 ./tezlib/lib/native/libhadoop.so.1.0.0 406850334 1572 -r-xr-xr-x 1 yarn hadoop 1607384 Sep 19 2018 ./tezlib/lib/native/libhadooppipes.a 406850331 24 -r-xr-xr-x 3 yarn hadoop 23800 Jun 10 2014 ./tezlib/lib/native/libsnappy.so.1 406850331 24 -r-xr-xr-x 3 yarn hadoop 23800 Jun 10 2014 ./tezlib/lib/native/libsnappy.so 406850332 280 -r-xr-xr-x 2 yarn hadoop 286676 Sep 19 2018 ./tezlib/lib/native/libhdfs.so 406850333 948 -r-xr-xr-x 2 yarn hadoop 968926 Sep 19 2018 ./tezlib/lib/native/libhadoop.so 406850335 452 -r-xr-xr-x 1 yarn hadoop 459400 Sep 19 2018 ./tezlib/lib/native/libhdfs.a 406850336 1676 -r-xr-xr-x 1 yarn hadoop 1712408 Sep 19 2018 ./tezlib/lib/native/libhadoop.a broken symlinks(find -L . -maxdepth 5 -type l -ls): End of LogType:directory.info ******************************************************************************* Container: container_e06_1555415227039_0004_01_000002 on ip-172-31-88-76.ec2.internal_45454_1555418368531 LogAggregationType: AGGREGATED ========================================================================================================= LogType:launch_container.sh LogLastModifiedTime:Tue Apr 16 12:39:28 +0000 2019 LogLength:5415 LogContents: #!/bin/bash set -o pipefail -e export PRELAUNCH_OUT="/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/prelaunch.out" exec >"${PRELAUNCH_OUT}" export PRELAUNCH_ERR="/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/prelaunch.err" exec 2>"${PRELAUNCH_ERR}" echo "Setting up env variables" export JAVA_HOME=${JAVA_HOME:-"/usr/java/default"} export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/usr/hdp/3.0.1.0-187/hadoop/conf"} export HADOOP_YARN_HOME=${HADOOP_YARN_HOME:-"/usr/hdp/3.0.1.0-187/hadoop-yarn"} export HADOOP_HOME=${HADOOP_HOME:-"/usr/hdp/3.0.1.0-187/hadoop"} export PATH=${PATH:-"/usr/sbin:/sbin:/usr/lib/ambari-server/*:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/var/lib/ambari-agent"} export LANG=${LANG:-"en_US.UTF-8"} export HADOOP_TOKEN_FILE_LOCATION="/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/container_tokens" export CONTAINER_ID="container_e06_1555415227039_0004_01_000002" export NM_PORT="45454" export NM_HOST="ip-172-31-88-76.ec2.internal" export NM_HTTP_PORT="8042" export LOCAL_DIRS="/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004" export LOCAL_USER_DIRS="/hadoop/yarn/local/usercache/hive/" export LOG_DIRS="/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002" export USER="hive" export LOGNAME="hive" export HOME="/home/" export PWD="/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002" export JVM_PID="$$" export MALLOC_ARENA_MAX="4" export NM_AUX_SERVICE_timeline_collector="" export NM_AUX_SERVICE_mapreduce_shuffle="AAA0+gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" export NM_AUX_SERVICE_spark2_shuffle="" export SHELL="/bin/bash" export HADOOP_ROOT_LOGGER="INFO,CLA" export CLASSPATH="/usr/hdp/3.0.1.0-187/hadoop/lib/hadoop-lzo-0.6.0.3.0.1.0-187.jar:/etc/hadoop/conf/secure:$PWD:$PWD/*:$PWD/tezlib/*:$PWD/tezlib/lib/*:" export LD_LIBRARY_PATH="$PWD:/usr/hdp/3.0.1.0-187/hadoop/lib/native:/usr/hdp/3.0.1.0-187/hadoop/lib/native/Linux-amd64-64:$PWD:/usr/hdp/3.0.1.0-187/hadoop/lib/native:/usr/hdp/3.0.1.0-187/hadoop/lib/native/Linux-amd64-64:$PWD:/usr/hdp/3.0.1.0-187/hadoop/lib/native:/usr/hdp/3.0.1.0-187/hadoop/lib/native/Linux-amd64-64:$PWD:/usr/hdp/3.0.1.0-187/hadoop/lib/native:/usr/hdp/3.0.1.0-187/hadoop/lib/native/Linux-amd64-64:$HADOOP_COMMON_HOME/lib/native/" echo "Setting up job resources" ln -sf "/hadoop/yarn/local/usercache/hive/filecache/16/TimestampUDF-1.0-SNAPSHOT.jar" "TimestampUDF-1.0-SNAPSHOT.jar" ln -sf "/hadoop/yarn/local/usercache/hive/filecache/10/hive-exec-3.1.0.3.0.1.0-187-ffcc945cef1d45a4f8c86dab4e04eaea5bb4284dae0788c8b7f4df27141566d9.jar" "hive-exec-3.1.0.3.0.1.0-187-ffcc945cef1d45a4f8c86dab4e04eaea5bb4284dae0788c8b7f4df27141566d9.jar" ln -sf "/hadoop/yarn/local/usercache/hive/filecache/15/hive-hcatalog-core.jar" "hive-hcatalog-core.jar" ln -sf "/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/filecache/10/tez-conf.pb" "tez-conf.pb" ln -sf "/hadoop/yarn/local/filecache/10/tez.tar.gz" "tezlib" echo "Copying debugging information" # Creating copy of launch script cp "launch_container.sh" "/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/launch_container.sh" chmod 640 "/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/launch_container.sh" # Determining directory contents echo "ls -l:" 1>"/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/directory.info" ls -l 1>>"/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/directory.info" echo "find -L . -maxdepth 5 -ls:" 1>>"/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/directory.info" find -L . -maxdepth 5 -ls 1>>"/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/directory.info" echo "broken symlinks(find -L . -maxdepth 5 -type l -ls):" 1>>"/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/directory.info" find -L . -maxdepth 5 -type l -ls 1>>"/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/directory.info" echo "Launching container" exec /bin/bash -c "$JAVA_HOME/bin/java -Xmx3276m -server -Djava.net.preferIPv4Stack=true -Dhdp.version=3.0.1.0-187 -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB -server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator -Dlog4j.configuration=tez-container-log4j.properties -Dyarn.app.container.log.dir=/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002 -Dtez.root.logger=INFO,CLA -Djava.io.tmpdir=$PWD/tmp org.apache.tez.runtime.task.TezChild ip-172-31-83-216.ec2.internal 45483 container_e06_1555415227039_0004_01_000002 application_1555415227039_0004 1 1>/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/stdout 2>/hadoop/yarn/log/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/stderr " End of LogType:launch_container.sh ************************************************************************************ End of LogType:prelaunch.err ****************************************************************************** Container: container_e06_1555415227039_0004_01_000002 on ip-172-31-88-76.ec2.internal_45454_1555418368531 LogAggregationType: AGGREGATED ========================================================================================================= LogType:prelaunch.out LogLastModifiedTime:Tue Apr 16 12:39:28 +0000 2019 LogLength:100 LogContents: Setting up env variables Setting up job resources Copying debugging information Launching container End of LogType:prelaunch.out ****************************************************************************** Container: container_e06_1555415227039_0004_01_000002 on ip-172-31-88-76.ec2.internal_45454_1555418368531 LogAggregationType: AGGREGATED ========================================================================================================= LogType:stderr LogLastModifiedTime:Tue Apr 16 12:39:28 +0000 2019 LogLength:188 LogContents: 2019-04-16 12:37:31 Starting to run new task attempt: attempt_1555415227039_0004_1_00_000000_0 2019-04-16 12:37:39 Completed running task attempt: attempt_1555415227039_0004_1_00_000000_0 End of LogType:stderr *********************************************************************** Container: container_e06_1555415227039_0004_01_000002 on ip-172-31-88-76.ec2.internal_45454_1555418368531 LogAggregationType: AGGREGATED ========================================================================================================= LogType:stdout LogLastModifiedTime:Tue Apr 16 12:39:28 +0000 2019 LogLength:18887 LogContents: 0.980: [GC pause (G1 Evacuation Pause) (young), 0.0104268 secs] [Parallel Time: 7.9 ms, GC Workers: 4] [GC Worker Start (ms): Min: 980.1, Avg: 980.2, Max: 980.6, Diff: 0.5] [Ext Root Scanning (ms): Min: 0.0, Avg: 0.7, Max: 2.3, Diff: 2.3, Sum: 2.8] [Update RS (ms): Min: 0.0, Avg: 0.2, Max: 1.0, Diff: 1.0, Sum: 1.0] [Processed Buffers: Min: 0, Avg: 0.0, Max: 0, Diff: 0, Sum: 0] [Scan RS (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [Code Root Scanning (ms): Min: 0.0, Avg: 3.0, Max: 6.0, Diff: 6.0, Sum: 12.1] [Object Copy (ms): Min: 0.0, Avg: 2.6, Max: 5.8, Diff: 5.8, Sum: 10.3] [Termination (ms): Min: 0.0, Avg: 0.6, Max: 1.0, Diff: 1.0, Sum: 2.5] [Termination Attempts: Min: 1, Avg: 1.5, Max: 2, Diff: 1, Sum: 6] [GC Worker Other (ms): Min: 0.0, Avg: 0.3, Max: 1.0, Diff: 1.0, Sum: 1.0] [GC Worker Total (ms): Min: 7.0, Avg: 7.4, Max: 7.6, Diff: 0.5, Sum: 29.7] [GC Worker End (ms): Min: 987.6, Avg: 987.6, Max: 987.7, Diff: 0.0] [Code Root Fixup: 0.0 ms] [Code Root Purge: 0.0 ms] [Clear CT: 0.1 ms] [Other: 2.4 ms] [Choose CSet: 0.0 ms] [Ref Proc: 2.2 ms] [Ref Enq: 0.0 ms] [Redirty Cards: 0.0 ms] [Humongous Register: 0.0 ms] [Humongous Reclaim: 0.0 ms] [Free CSet: 0.0 ms] [Eden: 26.0M(26.0M)->0.0B(22.0M) Survivors: 0.0B->4096.0K Heap: 26.0M(242.0M)->5117.0K(242.0M)] [Times: user=0.02 sys=0.00, real=0.01 secs] 1.394: [GC pause (G1 Evacuation Pause) (young), 0.0097542 secs] [Parallel Time: 8.8 ms, GC Workers: 4] [GC Worker Start (ms): Min: 1394.3, Avg: 1394.7, Max: 1395.6, Diff: 1.3] [Ext Root Scanning (ms): Min: 0.0, Avg: 0.6, Max: 1.0, Diff: 1.0, Sum: 2.4] [Update RS (ms): Min: 0.0, Avg: 0.1, Max: 0.2, Diff: 0.2, Sum: 0.3] [Processed Buffers: Min: 0, Avg: 0.5, Max: 1, Diff: 1, Sum: 2] [Scan RS (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [Code Root Scanning (ms): Min: 0.0, Avg: 2.1, Max: 7.4, Diff: 7.4, Sum: 8.3] [Object Copy (ms): Min: 0.0, Avg: 3.2, Max: 4.6, Diff: 4.6, Sum: 13.0] [Termination (ms): Min: 0.0, Avg: 2.3, Max: 3.1, Diff: 3.1, Sum: 9.2] [Termination Attempts: Min: 1, Avg: 1.0, Max: 1, Diff: 0, Sum: 4] [GC Worker Other (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [GC Worker Total (ms): Min: 7.4, Avg: 8.3, Max: 8.7, Diff: 1.3, Sum: 33.2] [GC Worker End (ms): Min: 1403.0, Avg: 1403.0, Max: 1403.0, Diff: 0.0] [Code Root Fixup: 0.0 ms] [Code Root Purge: 0.0 ms] [Clear CT: 0.1 ms] [Other: 0.9 ms] [Choose CSet: 0.0 ms] [Ref Proc: 0.7 ms] [Ref Enq: 0.0 ms] [Redirty Cards: 0.0 ms] [Humongous Register: 0.0 ms] [Humongous Reclaim: 0.0 ms] [Free CSet: 0.0 ms] [Eden: 22.0M(22.0M)->0.0B(22.0M) Survivors: 4096.0K->4096.0K Heap: 27.0M(242.0M)->8179.5K(242.0M)] [Times: user=0.03 sys=0.00, real=0.01 secs] 2019-04-16 12:37:31 Starting to run new task attempt: attempt_1555415227039_0004_1_00_000000_0 1.738: [GC pause (G1 Evacuation Pause) (young), 0.0114522 secs] [Parallel Time: 8.9 ms, GC Workers: 4] [GC Worker Start (ms): Min: 1738.6, Avg: 1739.3, Max: 1739.6, Diff: 1.0] [Ext Root Scanning (ms): Min: 0.8, Avg: 1.8, Max: 3.0, Diff: 2.2, Sum: 7.1] [Update RS (ms): Min: 0.0, Avg: 0.2, Max: 0.5, Diff: 0.5, Sum: 0.9] [Processed Buffers: Min: 0, Avg: 3.0, Max: 6, Diff: 6, Sum: 12] [Scan RS (ms): Min: 0.0, Avg: 0.1, Max: 0.2, Diff: 0.2, Sum: 0.3] [Code Root Scanning (ms): Min: 0.0, Avg: 1.3, Max: 5.0, Diff: 5.0, Sum: 5.2] [Object Copy (ms): Min: 0.5, Avg: 3.4, Max: 4.6, Diff: 4.1, Sum: 13.7] [Termination (ms): Min: 0.0, Avg: 0.5, Max: 0.7, Diff: 0.7, Sum: 1.9] [Termination Attempts: Min: 1, Avg: 1.0, Max: 1, Diff: 0, Sum: 4] [GC Worker Other (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [GC Worker Total (ms): Min: 7.0, Avg: 7.3, Max: 8.0, Diff: 1.0, Sum: 29.2] [GC Worker End (ms): Min: 1746.6, Avg: 1746.6, Max: 1746.7, Diff: 0.0] [Code Root Fixup: 0.0 ms] [Code Root Purge: 0.0 ms] [Clear CT: 0.1 ms] [Other: 2.5 ms] [Choose CSet: 0.0 ms] [Ref Proc: 2.3 ms] [Ref Enq: 0.0 ms] [Redirty Cards: 0.0 ms] [Humongous Register: 0.0 ms] [Humongous Reclaim: 0.0 ms] [Free CSet: 0.0 ms] [Eden: 22.0M(22.0M)->0.0B(22.0M) Survivors: 4096.0K->4096.0K Heap: 30.0M(242.0M)->11.5M(242.0M)] [Times: user=0.02 sys=0.00, real=0.02 secs] 1.914: [GC pause (G1 Evacuation Pause) (young), 0.0116417 secs] [Parallel Time: 10.1 ms, GC Workers: 4] [GC Worker Start (ms): Min: 1913.7, Avg: 1918.5, Max: 1923.8, Diff: 10.1] [Ext Root Scanning (ms): Min: 0.0, Avg: 0.5, Max: 1.4, Diff: 1.4, Sum: 1.9] [Update RS (ms): Min: 0.0, Avg: 0.3, Max: 0.6, Diff: 0.6, Sum: 1.2] [Processed Buffers: Min: 0, Avg: 2.5, Max: 6, Diff: 6, Sum: 10] [Scan RS (ms): Min: 0.0, Avg: 0.3, Max: 0.6, Diff: 0.6, Sum: 1.0] [Code Root Scanning (ms): Min: 0.0, Avg: 0.0, Max: 0.2, Diff: 0.2, Sum: 0.2] [Object Copy (ms): Min: 0.0, Avg: 3.7, Max: 6.9, Diff: 6.9, Sum: 14.7] [Termination (ms): Min: 0.0, Avg: 0.5, Max: 0.6, Diff: 0.6, Sum: 1.9] [Termination Attempts: Min: 1, Avg: 1.8, Max: 4, Diff: 3, Sum: 7] [GC Worker Other (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [GC Worker Total (ms): Min: 0.0, Avg: 5.2, Max: 10.1, Diff: 10.1, Sum: 21.0] [GC Worker End (ms): Min: 1923.8, Avg: 1923.8, Max: 1923.8, Diff: 0.0] [Code Root Fixup: 0.0 ms] [Code Root Purge: 0.0 ms] [Clear CT: 0.1 ms] [Other: 1.4 ms] [Choose CSet: 0.0 ms] [Ref Proc: 1.2 ms] [Ref Enq: 0.0 ms] [Redirty Cards: 0.0 ms] [Humongous Register: 0.0 ms] [Humongous Reclaim: 0.0 ms] [Free CSet: 0.0 ms] [Eden: 22.0M(22.0M)->0.0B(22.0M) Survivors: 4096.0K->4096.0K Heap: 33.5M(242.0M)->16.0M(242.0M)] [Times: user=0.03 sys=0.01, real=0.01 secs] 2.172: [GC pause (G1 Evacuation Pause) (young), 0.0065043 secs] [Parallel Time: 5.1 ms, GC Workers: 4] [GC Worker Start (ms): Min: 2172.0, Avg: 2173.4, Max: 2177.0, Diff: 5.0] [Ext Root Scanning (ms): Min: 0.0, Avg: 0.5, Max: 0.9, Diff: 0.9, Sum: 2.0] [Update RS (ms): Min: 0.0, Avg: 0.5, Max: 0.8, Diff: 0.8, Sum: 2.1] [Processed Buffers: Min: 0, Avg: 3.2, Max: 9, Diff: 9, Sum: 13] [Scan RS (ms): Min: 0.0, Avg: 0.1, Max: 0.2, Diff: 0.2, Sum: 0.4] [Code Root Scanning (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [Object Copy (ms): Min: 0.0, Avg: 2.0, Max: 2.7, Diff: 2.7, Sum: 8.0] [Termination (ms): Min: 0.0, Avg: 0.5, Max: 0.6, Diff: 0.6, Sum: 1.9] [Termination Attempts: Min: 1, Avg: 1.5, Max: 3, Diff: 2, Sum: 6] [GC Worker Other (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [GC Worker Total (ms): Min: 0.0, Avg: 3.6, Max: 5.0, Diff: 5.0, Sum: 14.5] [GC Worker End (ms): Min: 2177.0, Avg: 2177.0, Max: 2177.1, Diff: 0.0] [Code Root Fixup: 0.0 ms] [Code Root Purge: 0.0 ms] [Clear CT: 0.1 ms] [Other: 1.4 ms] [Choose CSet: 0.0 ms] [Ref Proc: 1.2 ms] [Ref Enq: 0.0 ms] [Redirty Cards: 0.0 ms] [Humongous Register: 0.0 ms] [Humongous Reclaim: 0.0 ms] [Free CSet: 0.0 ms] [Eden: 22.0M(22.0M)->0.0B(22.0M) Survivors: 4096.0K->4096.0K Heap: 38.0M(242.0M)->16.5M(242.0M)] [Times: user=0.02 sys=0.00, real=0.00 secs] 2.298: [GC pause (G1 Evacuation Pause) (young), 0.0115157 secs] [Parallel Time: 10.2 ms, GC Workers: 4] [GC Worker Start (ms): Min: 2298.4, Avg: 2300.7, Max: 2307.6, Diff: 9.2] [Ext Root Scanning (ms): Min: 0.0, Avg: 0.4, Max: 0.6, Diff: 0.6, Sum: 1.5] [Update RS (ms): Min: 0.0, Avg: 0.2, Max: 0.3, Diff: 0.3, Sum: 0.7] [Processed Buffers: Min: 0, Avg: 2.2, Max: 4, Diff: 4, Sum: 9] [Scan RS (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [Code Root Scanning (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [Object Copy (ms): Min: 0.0, Avg: 2.2, Max: 3.1, Diff: 3.1, Sum: 9.0] [Termination (ms): Min: 0.0, Avg: 4.1, Max: 5.5, Diff: 5.5, Sum: 16.4] [Termination Attempts: Min: 1, Avg: 1.0, Max: 1, Diff: 0, Sum: 4] [GC Worker Other (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [GC Worker Total (ms): Min: 0.0, Avg: 6.9, Max: 9.2, Diff: 9.2, Sum: 27.7] [GC Worker End (ms): Min: 2307.6, Avg: 2307.6, Max: 2307.6, Diff: 0.1] [Code Root Fixup: 0.0 ms] [Code Root Purge: 0.0 ms] [Clear CT: 0.1 ms] [Other: 1.2 ms] [Choose CSet: 0.0 ms] [Ref Proc: 1.0 ms] [Ref Enq: 0.0 ms] [Redirty Cards: 0.0 ms] [Humongous Register: 0.0 ms] [Humongous Reclaim: 0.0 ms] [Free CSet: 0.0 ms] [Eden: 22.0M(22.0M)->0.0B(23.0M) Survivors: 4096.0K->3072.0K Heap: 38.5M(242.0M)->17.9M(242.0M)] [Times: user=0.03 sys=0.00, real=0.02 secs] 2.404: [GC pause (G1 Evacuation Pause) (young), 0.0081538 secs] [Parallel Time: 4.6 ms, GC Workers: 4] [GC Worker Start (ms): Min: 2404.4, Avg: 2406.5, Max: 2408.9, Diff: 4.6] [Ext Root Scanning (ms): Min: 0.0, Avg: 0.5, Max: 1.0, Diff: 1.0, Sum: 1.9] [Update RS (ms): Min: 0.0, Avg: 0.8, Max: 1.7, Diff: 1.7, Sum: 3.3] [Processed Buffers: Min: 0, Avg: 4.0, Max: 11, Diff: 11, Sum: 16] [Scan RS (ms): Min: 0.0, Avg: 0.1, Max: 0.2, Diff: 0.2, Sum: 0.3] [Code Root Scanning (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [Object Copy (ms): Min: 0.0, Avg: 0.3, Max: 0.6, Diff: 0.6, Sum: 1.1] [Termination (ms): Min: 0.0, Avg: 0.8, Max: 1.2, Diff: 1.2, Sum: 3.1] [Termination Attempts: Min: 1, Avg: 1.0, Max: 1, Diff: 0, Sum: 4] [GC Worker Other (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [GC Worker Total (ms): Min: 0.0, Avg: 2.4, Max: 4.6, Diff: 4.6, Sum: 9.8] [GC Worker End (ms): Min: 2408.9, Avg: 2408.9, Max: 2408.9, Diff: 0.0] [Code Root Fixup: 0.0 ms] [Code Root Purge: 0.0 ms] [Clear CT: 0.1 ms] [Other: 3.5 ms] [Choose CSet: 0.0 ms] [Ref Proc: 3.4 ms] [Ref Enq: 0.0 ms] [Redirty Cards: 0.0 ms] [Humongous Register: 0.0 ms] [Humongous Reclaim: 0.0 ms] [Free CSet: 0.0 ms] [Eden: 23.0M(23.0M)->0.0B(23.0M) Survivors: 3072.0K->3072.0K Heap: 40.9M(242.0M)->17.6M(242.0M)] [Times: user=0.02 sys=0.00, real=0.01 secs] 2.429: [GC pause (Metadata GC Threshold) (young) (initial-mark), 0.0037193 secs] [Parallel Time: 2.8 ms, GC Workers: 4] [GC Worker Start (ms): Min: 2428.6, Avg: 2429.2, Max: 2429.7, Diff: 1.1] [Ext Root Scanning (ms): Min: 0.0, Avg: 1.2, Max: 2.0, Diff: 2.0, Sum: 4.6] [Update RS (ms): Min: 0.0, Avg: 0.2, Max: 0.4, Diff: 0.4, Sum: 0.7] [Processed Buffers: Min: 0, Avg: 1.8, Max: 6, Diff: 6, Sum: 7] [Scan RS (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [Code Root Scanning (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [Object Copy (ms): Min: 0.0, Avg: 0.3, Max: 0.5, Diff: 0.5, Sum: 1.2] [Termination (ms): Min: 0.0, Avg: 0.5, Max: 0.6, Diff: 0.6, Sum: 1.8] [Termination Attempts: Min: 1, Avg: 1.0, Max: 1, Diff: 0, Sum: 4] [GC Worker Other (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [GC Worker Total (ms): Min: 1.5, Avg: 2.1, Max: 2.8, Diff: 1.3, Sum: 8.3] [GC Worker End (ms): Min: 2431.2, Avg: 2431.3, Max: 2431.4, Diff: 0.2] [Code Root Fixup: 0.0 ms] [Code Root Purge: 0.0 ms] [Clear CT: 0.0 ms] [Other: 0.8 ms] [Choose CSet: 0.0 ms] [Ref Proc: 0.7 ms] [Ref Enq: 0.0 ms] [Redirty Cards: 0.0 ms] [Humongous Register: 0.0 ms] [Humongous Reclaim: 0.0 ms] [Free CSet: 0.0 ms] [Eden: 4096.0K(23.0M)->0.0B(24.0M) Survivors: 3072.0K->2048.0K Heap: 20.9M(242.0M)->16.6M(242.0M)] [Times: user=0.01 sys=0.00, real=0.01 secs] 2.432: [GC concurrent-root-region-scan-start] 2.433: [GC concurrent-root-region-scan-end, 0.0009190 secs] 2.433: [GC concurrent-mark-start] 2.457: [GC concurrent-mark-end, 0.0232634 secs] 2.457: [GC remark 2.457: [Finalize Marking, 0.0006966 secs] 2.457: [GC ref-proc, 0.0000942 secs] 2.457: [Unloading, 0.0033721 secs], 0.0042759 secs] [Times: user=0.01 sys=0.00, real=0.01 secs] 2.461: [GC cleanup 22M->18M(242M), 0.0004457 secs] [Times: user=0.00 sys=0.00, real=0.00 secs] 2.463: [GC concurrent-cleanup-start] 2.463: [GC concurrent-cleanup-end, 0.0000195 secs] 2.679: [GC pause (G1 Evacuation Pause) (young), 0.0051780 secs] [Parallel Time: 3.7 ms, GC Workers: 4] [GC Worker Start (ms): Min: 2679.0, Avg: 2679.9, Max: 2682.7, Diff: 3.7] [Ext Root Scanning (ms): Min: 0.0, Avg: 0.7, Max: 0.9, Diff: 0.9, Sum: 2.7] [Update RS (ms): Min: 0.0, Avg: 0.4, Max: 0.8, Diff: 0.8, Sum: 1.7] [Processed Buffers: Min: 0, Avg: 3.2, Max: 7, Diff: 7, Sum: 13] [Scan RS (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [Code Root Scanning (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.1] [Object Copy (ms): Min: 0.0, Avg: 1.2, Max: 1.7, Diff: 1.7, Sum: 4.7] [Termination (ms): Min: 0.0, Avg: 0.5, Max: 0.6, Diff: 0.6, Sum: 1.9] [Termination Attempts: Min: 1, Avg: 1.2, Max: 2, Diff: 1, Sum: 5] [GC Worker Other (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [GC Worker Total (ms): Min: 0.0, Avg: 2.8, Max: 3.7, Diff: 3.7, Sum: 11.1] [GC Worker End (ms): Min: 2682.7, Avg: 2682.7, Max: 2682.7, Diff: 0.0] [Code Root Fixup: 0.0 ms] [Code Root Purge: 0.0 ms] [Clear CT: 0.1 ms] [Other: 1.4 ms] [Choose CSet: 0.0 ms] [Ref Proc: 1.2 ms] [Ref Enq: 0.0 ms] [Redirty Cards: 0.0 ms] [Humongous Register: 0.0 ms] [Humongous Reclaim: 0.0 ms] [Free CSet: 0.0 ms] [Eden: 24.0M(24.0M)->0.0B(22.0M) Survivors: 2048.0K->4096.0K Heap: 36.6M(242.0M)->14.2M(242.0M)] [Times: user=0.02 sys=0.00, real=0.00 secs] 3.033: [GC pause (G1 Evacuation Pause) (young), 0.0067295 secs] [Parallel Time: 5.0 ms, GC Workers: 4] [GC Worker Start (ms): Min: 3032.9, Avg: 3032.9, Max: 3032.9, Diff: 0.0] [Ext Root Scanning (ms): Min: 0.8, Avg: 2.0, Max: 4.9, Diff: 4.1, Sum: 8.0] [Update RS (ms): Min: 0.0, Avg: 0.4, Max: 0.8, Diff: 0.8, Sum: 1.8] [Processed Buffers: Min: 0, Avg: 2.2, Max: 4, Diff: 4, Sum: 9] [Scan RS (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [Code Root Scanning (ms): Min: 0.0, Avg: 0.1, Max: 0.1, Diff: 0.1, Sum: 0.2] [Object Copy (ms): Min: 0.0, Avg: 2.0, Max: 2.9, Diff: 2.9, Sum: 7.8] [Termination (ms): Min: 0.0, Avg: 0.5, Max: 0.6, Diff: 0.6, Sum: 1.9] [Termination Attempts: Min: 1, Avg: 1.0, Max: 1, Diff: 0, Sum: 4] [GC Worker Other (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [GC Worker Total (ms): Min: 4.9, Avg: 4.9, Max: 5.0, Diff: 0.0, Sum: 19.8] [GC Worker End (ms): Min: 3037.8, Avg: 3037.8, Max: 3037.8, Diff: 0.0] [Code Root Fixup: 0.0 ms] [Code Root Purge: 0.0 ms] [Clear CT: 0.1 ms] [Other: 1.7 ms] [Choose CSet: 0.0 ms] [Ref Proc: 1.5 ms] [Ref Enq: 0.0 ms] [Redirty Cards: 0.0 ms] [Humongous Register: 0.0 ms] [Humongous Reclaim: 0.0 ms] [Free CSet: 0.0 ms] [Eden: 22.0M(22.0M)->0.0B(22.0M) Survivors: 4096.0K->4096.0K Heap: 36.2M(242.0M)->16.4M(242.0M)] [Times: user=0.02 sys=0.00, real=0.01 secs] 3.292: [GC pause (G1 Evacuation Pause) (young), 0.0071040 secs] [Parallel Time: 5.3 ms, GC Workers: 4] [GC Worker Start (ms): Min: 3291.6, Avg: 3292.6, Max: 3294.4, Diff: 2.8] [Ext Root Scanning (ms): Min: 0.0, Avg: 0.8, Max: 1.4, Diff: 1.4, Sum: 3.3] [Update RS (ms): Min: 0.0, Avg: 0.5, Max: 0.8, Diff: 0.8, Sum: 2.0] [Processed Buffers: Min: 0, Avg: 3.0, Max: 6, Diff: 6, Sum: 12] [Scan RS (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [Code Root Scanning (ms): Min: 0.0, Avg: 0.0, Max: 0.1, Diff: 0.1, Sum: 0.1] [Object Copy (ms): Min: 2.5, Avg: 3.0, Max: 3.2, Diff: 0.7, Sum: 11.8] [Termination (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [Termination Attempts: Min: 1, Avg: 2.2, Max: 4, Diff: 3, Sum: 9] [GC Worker Other (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [GC Worker Total (ms): Min: 2.5, Avg: 4.3, Max: 5.3, Diff: 2.8, Sum: 17.3] [GC Worker End (ms): Min: 3296.9, Avg: 3296.9, Max: 3296.9, Diff: 0.0] [Code Root Fixup: 0.0 ms] [Code Root Purge: 0.0 ms] [Clear CT: 0.1 ms] [Other: 1.7 ms] [Choose CSet: 0.0 ms] [Ref Proc: 1.6 ms] [Ref Enq: 0.0 ms] [Redirty Cards: 0.1 ms] [Humongous Register: 0.0 ms] [Humongous Reclaim: 0.0 ms] [Free CSet: 0.0 ms] [Eden: 22.0M(22.0M)->0.0B(22.0M) Survivors: 4096.0K->4096.0K Heap: 38.4M(242.0M)->19.2M(242.0M)] [Times: user=0.02 sys=0.00, real=0.01 secs] 7.194: [GC pause (G1 Evacuation Pause) (young), 0.0066681 secs] [Parallel Time: 4.4 ms, GC Workers: 4] [GC Worker Start (ms): Min: 7194.0, Avg: 7194.0, Max: 7194.0, Diff: 0.0] [Ext Root Scanning (ms): Min: 0.6, Avg: 0.8, Max: 1.1, Diff: 0.5, Sum: 3.0] [Update RS (ms): Min: 1.2, Avg: 1.3, Max: 1.5, Diff: 0.3, Sum: 5.3] [Processed Buffers: Min: 3, Avg: 4.5, Max: 6, Diff: 3, Sum: 18] [Scan RS (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [Code Root Scanning (ms): Min: 0.0, Avg: 0.0, Max: 0.1, Diff: 0.1, Sum: 0.1] [Object Copy (ms): Min: 2.1, Avg: 2.2, Max: 2.3, Diff: 0.3, Sum: 8.7] [Termination (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [Termination Attempts: Min: 1, Avg: 1.8, Max: 3, Diff: 2, Sum: 7] [GC Worker Other (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0] [GC Worker Total (ms): Min: 4.3, Avg: 4.3, Max: 4.3, Diff: 0.0, Sum: 17.2] [GC Worker End (ms): Min: 7198.3, Avg: 7198.3, Max: 7198.3, Diff: 0.0] [Code Root Fixup: 0.0 ms] [Code Root Purge: 0.0 ms] [Clear CT: 0.1 ms] [Other: 2.3 ms] [Choose CSet: 0.0 ms] [Ref Proc: 2.1 ms] [Ref Enq: 0.0 ms] [Redirty Cards: 0.0 ms] [Humongous Register: 0.0 ms] [Humongous Reclaim: 0.0 ms] [Free CSet: 0.0 ms] [Eden: 22.0M(22.0M)->0.0B(22.0M) Survivors: 4096.0K->4096.0K Heap: 41.2M(242.0M)->22.4M(242.0M)] [Times: user=0.02 sys=0.00, real=0.01 secs] 2019-04-16 12:37:39 Completed running task attempt: attempt_1555415227039_0004_1_00_000000_0 Heap garbage-first heap total 247808K, used 39372K [0x00000006f3400000, 0x00000006f3500790, 0x00000007c0000000) region size 1024K, 21 young (21504K), 4 survivors (4096K) Metaspace used 33457K, capacity 33737K, committed 34048K, reserved 1079296K class space used 3878K, capacity 3983K, committed 4096K, reserved 1048576K End of LogType:stdout *********************************************************************** Container: container_e06_1555415227039_0004_01_000002 on ip-172-31-88-76.ec2.internal_45454_1555418368531 LogAggregationType: AGGREGATED ========================================================================================================= LogType:syslog LogLastModifiedTime:Tue Apr 16 12:39:28 +0000 2019 LogLength:27183 LogContents: 2019-04-16 12:37:30,815 [INFO] [main] |task.TezChild|: TezChild starting with PID=18519, containerIdentifier=container_e06_1555415227039_0004_01_000002 2019-04-16 12:37:31,295 [INFO] [main] |shim.HadoopShimsLoader|: Trying to locate HadoopShimProvider for hadoopVersion=3.1.1.3.0.1.0-187, majorVersion=3, minorVersion=1 2019-04-16 12:37:31,297 [INFO] [main] |shim.HadoopShimsLoader|: Picked HadoopShim org.apache.tez.hadoop.shim.DefaultHadoopShim, providerName=null, overrideProviderViaConfig=null, hadoopVersion=3.1.1.3.0.1.0-187, majorVersion=3, minorVersion=1 2019-04-16 12:37:31,336 [INFO] [main] |task.TezChild|: /************************************************************ [system properties] os.name: Linux os.version: 3.10.0-693.5.2.el7.x86_64 java.home: /usr/java/jdk1.8.0_161/jre java.runtime.version: 1.8.0_161-b12 java.vendor: Oracle Corporation java.version: 1.8.0_161 java.vm.name: Java HotSpot(TM) 64-Bit Server VM java.class.path: /usr/hdp/3.0.1.0-187/hadoop/lib/hadoop-lzo-0.6.0.3.0.1.0-187.jar:/etc/hadoop/conf/secure:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/TimestampUDF-1.0-SNAPSHOT.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/hive-exec-3.1.0.3.0.1.0-187-ffcc945cef1d45a4f8c86dab4e04eaea5bb4284dae0788c8b7f4df27141566d9.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/hive-hcatalog-core.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/hadoop-shim-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/tez-api-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/tez-common-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/tez-runtime-internals-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/tez-runtime-library-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/tez-mapreduce-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/tez-examples-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/tez-dag-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/tez-tests-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/tez-ext-service-tests-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/tez-protobuf-history-plugin-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/tez-yarn-timeline-history-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/tez-yarn-timeline-history-with-acls-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/tez-yarn-timeline-cache-plugin-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/tez-yarn-timeline-history-with-fs-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/tez-history-parser-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/tez-job-analyzer-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/tez-javadoc-tools-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/hadoop-shim-2.8-0.9.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/slf4j-api-1.7.10.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/hadoop-yarn-api-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/commons-lang-2.6.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/guava-11.0.2.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jsr305-3.0.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/commons-logging-1.1.3.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jaxb-api-2.2.11.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/protobuf-java-2.5.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jackson-annotations-2.9.5.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/hadoop-common-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/hadoop-annotations-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/commons-cli-1.2.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/commons-math3-3.1.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/httpclient-4.5.2.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/httpcore-4.4.4.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/commons-codec-1.4.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/commons-io-2.4.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/commons-net-3.6.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/commons-collections-3.2.2.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/javax.servlet-api-3.1.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jetty-servlet-9.3.19.v20170502.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jetty-security-9.3.19.v20170502.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jetty-webapp-9.3.19.v20170502.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jetty-xml-9.3.19.v20170502.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jersey-servlet-1.19.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jersey-json-1.19.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jettison-1.3.4.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jaxb-impl-2.2.3-1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jackson-core-asl-1.9.2.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jackson-mapper-asl-1.9.2.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jackson-jaxrs-1.9.2.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jackson-xc-1.9.2.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/log4j-1.2.17.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/commons-beanutils-1.9.3.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/commons-configuration2-2.1.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/commons-lang3-3.4.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/slf4j-log4j12-1.7.10.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/avro-1.7.7.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/paranamer-2.3.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/snappy-java-1.0.5.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/commons-compress-1.4.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/xz-1.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/re2j-1.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/gson-2.2.4.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/hadoop-auth-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/nimbus-jose-jwt-4.41.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jcip-annotations-1.0-1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/json-smart-2.3.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/accessors-smart-1.2.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/asm-5.0.4.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/zookeeper-3.4.6.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/netty-3.10.5.Final.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/curator-framework-2.12.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/curator-client-2.12.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/kerb-simplekdc-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/kerb-client-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/kerby-config-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/kerb-core-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/kerby-pkix-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/kerby-asn1-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/kerby-util-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/kerb-common-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/kerb-crypto-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/kerb-util-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/token-provider-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/kerb-admin-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/kerb-server-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/kerb-identity-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/kerby-xdr-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/curator-recipes-2.12.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/htrace-core4-4.1.0-incubating.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jline-0.9.94.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jackson-databind-2.9.5.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jackson-core-2.9.5.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/stax2-api-3.1.4.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/woodstox-core-5.0.3.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/hadoop-aws-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/aws-java-sdk-bundle-1.11.271.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/netty-codec-http-4.1.17.Final.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/netty-codec-4.1.17.Final.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/netty-transport-4.1.17.Final.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/netty-buffer-4.1.17.Final.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/netty-common-4.1.17.Final.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/netty-resolver-4.1.17.Final.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/netty-handler-4.1.17.Final.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/hadoop-azure-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/azure-storage-7.0.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/azure-keyvault-core-1.0.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/guice-4.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/javax.inject-1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/aopalliance-1.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jetty-util-ajax-9.3.19.v20170502.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jetty-util-9.3.22.v20171030.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/wildfly-openssl-1.0.4.Final.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/hadoop-azure-datalake-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/azure-data-lake-store-sdk-2.2.7.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/gcs-connector-1.9.0.3.0.1.0-187-shaded.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/stax-api-1.0.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/hadoop-yarn-common-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jersey-core-1.19.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jsr311-api-1.1.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jersey-client-1.19.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/guice-servlet-4.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jersey-guice-1.19.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jackson-module-jaxb-annotations-2.9.5.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jackson-jaxrs-json-provider-2.9.5.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jackson-jaxrs-base-2.9.5.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/hadoop-yarn-client-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/commons-collections4-4.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jetty-server-9.3.22.v20171030.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jetty-http-9.3.22.v20171030.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/jetty-io-9.3.22.v20171030.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/commons-daemon-1.0.13.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/leveldbjni-all-1.8.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/hadoop-hdfs-client-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/okhttp-2.7.5.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/okio-1.6.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/RoaringBitmap-0.4.9.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/async-http-client-1.9.40.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/hadoop-mapreduce-client-core-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/hadoop-mapreduce-client-common-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/hadoop-yarn-server-web-proxy-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/hadoop-yarn-server-common-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/hadoop-yarn-registry-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/dnsjava-2.1.7.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/geronimo-jcache_1.0_spec-1.0-alpha-1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/ehcache-3.3.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/HikariCP-java7-2.4.12.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/mssql-jdbc-6.2.1.jre7.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/servlet-api-2.5.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/metrics-core-3.1.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/hadoop-yarn-server-applicationhistoryservice-3.1.1.3.0.1.0-187.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/objenesis-1.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/fst-2.50.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/java-util-1.9.0.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/json-io-2.5.1.jar:/hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tezlib/lib/hadoop-yarn-server-timeline-pluginstorage-3.1.1.3.0.1.0-187.jar: java.io.tmpdir: /hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002/tmp user.dir: /hadoop/yarn/local/usercache/hive/appcache/application_1555415227039_0004/container_e06_1555415227039_0004_01_000002 user.name: yarn ************************************************************/ 2019-04-16 12:37:31,346 [INFO] [main] |security.SecurityUtil|: Updating Configuration 2019-04-16 12:37:31,427 [INFO] [TezChild] |task.ContainerReporter|: Attempting to fetch new task for container container_e06_1555415227039_0004_01_000002 2019-04-16 12:37:31,660 [INFO] [TezChild] |task.ContainerReporter|: Got TaskUpdate for containerId= container_e06_1555415227039_0004_01_000002: 0 ms after starting to poll. TaskInfo: shouldDie: false, currentTaskAttemptId: attempt_1555415227039_0004_1_00_000000_0 2019-04-16 12:37:31,663 [INFO] [main] |common.TezUtilsInternal|: Redirecting log file based on addend: attempt_1555415227039_0004_1_00_000000_0 2019-04-16 12:37:39,815 [INFO] [TezChild] |task.ContainerReporter|: Attempting to fetch new task for container container_e06_1555415227039_0004_01_000002 End of LogType:syslog *********************************************************************** Container: container_e06_1555415227039_0004_01_000002 on ip-172-31-88-76.ec2.internal_45454_1555418368531 LogAggregationType: AGGREGATED ========================================================================================================= LogType:syslog_attempt_1555415227039_0004_1_00_000000_0 LogLastModifiedTime:Tue Apr 16 12:39:28 +0000 2019 LogLength:14927 LogContents: 2019-04-16 12:37:31,672 [INFO] [main] |task.TezChild|: Refreshing UGI since Credentials have changed. Credentials : #Tokens=1, #SecretKeys=0 2019-04-16 12:37:31,701 [INFO] [main] |metrics.TaskCounterUpdater|: Using ResourceCalculatorProcessTree : org.apache.tez.util.TezMxBeanResourceCalculator 2019-04-16 12:37:31,702 [INFO] [main] |runtime.LogicalIOProcessorRuntimeTask|: Initializing LogicalIOProcessorRuntimeTask with TaskSpec: DAGName : Select T.Candidate.startTime, T.Candidat...T (Stage-1), VertexName: Map 1, VertexParallelism: 1, TaskAttemptID:attempt_1555415227039_0004_1_00_000000_0, processorName=org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor, inputSpecListSize=1, outputSpecListSize=1, inputSpecList=[{{ sourceVertexName=idtable, physicalEdgeCount=1, inputClassName=org.apache.tez.mapreduce.input.MRInputLegacy }}, ], outputSpecList=[{{ destinationVertexName=out_Map 1, physicalEdgeCount=0, outputClassName=org.apache.tez.mapreduce.output.MROutput }}, ], taskConfEntryCount=0 2019-04-16 12:37:31,703 [INFO] [main] |resources.MemoryDistributor|: InitialMemoryDistributor (isEnabled=true) invoked with: numInputs=1, numOutputs=1, JVM.maxFree=3435134976, allocatorClassName=org.apache.tez.runtime.library.resources.WeightedScalingMemoryDistributor 2019-04-16 12:37:31,715 [INFO] [TezChild] |task.TaskRunner2Callable|: Initializing task, taskAttemptId=attempt_1555415227039_0004_1_00_000000_0 2019-04-16 12:37:31,738 [INFO] [TaskHeartbeatThread] |task.TaskReporter|: Routing events from heartbeat response to task, currentTaskAttemptId=attempt_1555415227039_0004_1_00_000000_0, eventCount=1 fromEventId=0 nextFromEventId=0 2019-04-16 12:37:31,752 [INFO] [TezChild] |Configuration.deprecation|: mapred.committer.job.setup.cleanup.needed is deprecated. Instead, use mapreduce.job.committer.setup.cleanup.needed 2019-04-16 12:37:31,753 [INFO] [TezChild] |Configuration.deprecation|: mapred.input.dir.recursive is deprecated. Instead, use mapreduce.input.fileinputformat.input.dir.recursive 2019-04-16 12:37:31,760 [INFO] [TezChild] |Configuration.deprecation|: mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id 2019-04-16 12:37:31,760 [INFO] [TezChild] |Configuration.deprecation|: mapred.task.partition is deprecated. Instead, use mapreduce.task.partition 2019-04-16 12:37:31,760 [INFO] [TezChild] |runtime.LogicalIOProcessorRuntimeTask|: Initialized processor 2019-04-16 12:37:31,760 [INFO] [TezChild] |runtime.LogicalIOProcessorRuntimeTask|: Waiting for 2 initializers to finish 2019-04-16 12:37:31,821 [INFO] [I/O Setup 0 Initialize: {idtable}] |counters.Limits|: Counter limits initialized with parameters: GROUP_NAME_MAX=256, MAX_GROUPS=3000, COUNTER_NAME_MAX=64, MAX_COUNTERS=10000 2019-04-16 12:37:31,826 [INFO] [I/O Setup 0 Initialize: {idtable}] |input.MRInput|: idtable using newmapreduce API=false, split via event=true, numPhysicalInputs=1 2019-04-16 12:37:31,826 [INFO] [I/O Setup 0 Initialize: {idtable}] |input.MRInputLegacy|: idtable MRInputLegacy deferring initialization 2019-04-16 12:37:31,826 [INFO] [TezChild] |runtime.LogicalIOProcessorRuntimeTask|: Waiting for 1 initializers to finish 2019-04-16 12:37:31,834 [INFO] [I/O Setup 1 Initialize: {out_Map 1}] |output.MROutput|: out_Map 1: outputFormat=org.apache.hadoop.hive.ql.io.HiveOutputFormatImpl, using newmapreduce API=false 2019-04-16 12:37:32,597 [INFO] [TezChild] |runtime.LogicalIOProcessorRuntimeTask|: All initializers finished 2019-04-16 12:37:32,603 [INFO] [TezChild] |resources.MemoryDistributor|: InitialRequests=[idtable:INPUT:0:org.apache.tez.mapreduce.input.MRInputLegacy], [out_Map 1:OUTPUT:0:org.apache.tez.mapreduce.output.MROutput] 2019-04-16 12:37:32,623 [INFO] [TezChild] |resources.WeightedScalingMemoryDistributor|: ScaleRatiosUsed=[PARTITIONED_UNSORTED_OUTPUT:1][UNSORTED_OUTPUT:1][UNSORTED_INPUT:1][SORTED_OUTPUT:12][SORTED_MERGED_INPUT:12][PROCESSOR:1][OTHER:1] 2019-04-16 12:37:32,631 [INFO] [TezChild] |resources.WeightedScalingMemoryDistributor|: InitialReservationFraction=0.3, AdditionalReservationFractionForIOs=0.03, finalReserveFractionUsed=0.32999999999999996 2019-04-16 12:37:32,631 [INFO] [TezChild] |resources.WeightedScalingMemoryDistributor|: Scaling Requests. NumRequests: 2, numScaledRequests: 2, TotalRequested: 0, TotalRequestedScaled: 0.0, TotalJVMHeap: 3435134976, TotalAvailable: 2301540433, TotalRequested/TotalJVMHeap:0.00 2019-04-16 12:37:32,632 [INFO] [TezChild] |resources.MemoryDistributor|: Allocations=[idtable:org.apache.tez.mapreduce.input.MRInputLegacy:INPUT:0:0], [out_Map 1:org.apache.tez.mapreduce.output.MROutput:OUTPUT:0:0] 2019-04-16 12:37:32,632 [INFO] [TezChild] |runtime.LogicalIOProcessorRuntimeTask|: Starting Inputs/Outputs 2019-04-16 12:37:32,632 [INFO] [TezChild] |runtime.LogicalIOProcessorRuntimeTask|: Input: idtable being auto started by the framework. Subsequent instances will not be auto-started 2019-04-16 12:37:32,632 [INFO] [I/O Setup 0 Start: {idtable}] |runtime.LogicalIOProcessorRuntimeTask|: Started Input with src edge: idtable 2019-04-16 12:37:32,632 [INFO] [TezChild] |runtime.LogicalIOProcessorRuntimeTask|: Num IOs determined for AutoStart: 1 2019-04-16 12:37:32,632 [INFO] [TezChild] |runtime.LogicalIOProcessorRuntimeTask|: Waiting for 1 IOs to start 2019-04-16 12:37:32,632 [INFO] [TezChild] |runtime.LogicalIOProcessorRuntimeTask|: AutoStartComplete 2019-04-16 12:37:32,633 [INFO] [TezChild] |task.TaskRunner2Callable|: Running task, taskAttemptId=attempt_1555415227039_0004_1_00_000000_0 2019-04-16 12:37:32,703 [INFO] [TezChild] |conf.HiveConf|: Found configuration file null 2019-04-16 12:37:32,893 [INFO] [TezChild] |exec.Utilities|: PLAN PATH = hdfs://ip-172-31-69-167.ec2.internal:8020/tmp/hive/hive/4adc7ed2-01cc-481b-b2aa-bbe41551ac08/hive_2019-04-16_12-37-02_586_6615575765504587239-2/hive/_tez_scratch_dir/8187bcb8-4a61-4ade-8498-5577705993a2/map.xml 2019-04-16 12:37:32,896 [INFO] [TezChild] |exec.SerializationUtilities|: Deserializing MapWork using kryo 2019-04-16 12:37:33,128 [INFO] [TezChild] |exec.Utilities|: Deserialized plan (via RPC) - name: Map 1 size: 3.14KB 2019-04-16 12:37:33,128 [INFO] [TezChild] |tez.ObjectCache|: Caching key: hive_20190416123702_5ee0ce4b-526b-4828-b572-3bb9507e0f85_Map 1__MAP_PLAN__ 2019-04-16 12:37:33,132 [INFO] [TezChild] |tez.MapRecordProcessor|: The input names are: [idtable] 2019-04-16 12:37:33,133 [INFO] [TezChild] |input.MRInput|: Initialized MRInput: idtable 2019-04-16 12:37:33,151 [INFO] [TezChild] |exec.Utilities|: PLAN PATH = hdfs://ip-172-31-69-167.ec2.internal:8020/tmp/hive/hive/4adc7ed2-01cc-481b-b2aa-bbe41551ac08/hive_2019-04-16_12-37-02_586_6615575765504587239-2/hive/_tez_scratch_dir/8187bcb8-4a61-4ade-8498-5577705993a2/map.xml 2019-04-16 12:37:33,169 [INFO] [TezChild] |exec.Utilities|: PLAN PATH = hdfs://ip-172-31-69-167.ec2.internal:8020/tmp/hive/hive/4adc7ed2-01cc-481b-b2aa-bbe41551ac08/hive_2019-04-16_12-37-02_586_6615575765504587239-2/hive/_tez_scratch_dir/8187bcb8-4a61-4ade-8498-5577705993a2/map.xml 2019-04-16 12:37:33,171 [INFO] [TezChild] |exec.Utilities|: PLAN PATH = hdfs://ip-172-31-69-167.ec2.internal:8020/tmp/hive/hive/4adc7ed2-01cc-481b-b2aa-bbe41551ac08/hive_2019-04-16_12-37-02_586_6615575765504587239-2/hive/_tez_scratch_dir/8187bcb8-4a61-4ade-8498-5577705993a2/map.xml 2019-04-16 12:37:33,426 [INFO] [TezChild] |impl.OrcCodecPool|: Got brand-new codec ZLIB 2019-04-16 12:37:33,458 [INFO] [TezChild] |orc.ReaderImpl|: Reading ORC rows from hdfs://ip-172-31-69-167.ec2.internal:8020/warehouse/tablespace/managed/hive/idtable/base_0000001/bucket_00000 with {include: [true, false], offset: 3, length: 276, schema: struct, includeAcidColumns: true} 2019-04-16 12:37:33,494 [INFO] [TezChild] |lib.MRReaderMapred|: Processing split: TezGroupedSplit{wrappedSplits=[org.apache.hadoop.hive.ql.io.orc.OrcInputFormat:OrcSplit [hdfs://ip-172-31-69-167.ec2.internal:8020/warehouse/tablespace/managed/hive/idtable/base_0000001/bucket_00000, start=3, length=276, isOriginal=false, fileLength=613, hasFooter=false, hasBase=true, deltas=0]], wrappedInputFormatName='org.apache.hadoop.hive.ql.io.HiveInputFormat', locations=[ip-172-31-93-195.ec2.internal, ip-172-31-88-76.ec2.internal, ip-172-31-83-216.ec2.internal], rack='null', length=276} 2019-04-16 12:37:33,498 [INFO] [TezChild] |input.MRInput|: idtable initialized RecordReader from event 2019-04-16 12:37:33,501 [INFO] [TezChild] |tez.MapRecordProcessor|: Main input name is Map 1 2019-04-16 12:37:33,501 [INFO] [TezChild] |vector.VectorMapOperator|: Initializing operator MAP[0] 2019-04-16 12:37:33,503 [INFO] [TezChild] |tez.MapRecordProcessor|: Id =0 null <\Children> <\Parent> <\MAP> 2019-04-16 12:37:33,505 [INFO] [TezChild] |tez.MapRecordProcessor|: Memory available for operators set to 3.20GB 2019-04-16 12:37:33,511 [INFO] [TezChild] |tez.ObjectCache|: Caching key: hive_20190416123702_5ee0ce4b-526b-4828-b572-3bb9507e0f85_DynamicValueRegistry 2019-04-16 12:37:33,513 [INFO] [TezChild] |exec.TableScanOperator|: Initializing operator TS[0] 2019-04-16 12:37:33,513 [INFO] [TezChild] |vector.VectorSelectOperator|: Initializing operator SEL[3] 2019-04-16 12:37:33,521 [INFO] [TezChild] |vector.VectorFileSinkOperator|: Initializing operator FS[4] 2019-04-16 12:37:33,583 [INFO] [TezChild] |exec.FileSinkOperator|: Using serializer : class org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe[[[B@66173f4b]:[_col0, _col1, _col2]:[string, string, bigint]] and formatter : org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat@1a4b45fc 2019-04-16 12:37:33,583 [INFO] [TezChild] |Configuration.deprecation|: mapred.healthChecker.script.timeout is deprecated. Instead, use mapreduce.tasktracker.healthchecker.script.timeout 2019-04-16 12:37:33,592 [INFO] [TezChild] |hive.TimestampUDF|: == TimestampUDF == 2019-04-16 12:37:33,594 [INFO] [TezChild] |hive.TimestampUDF|: startTime: 2019-04-16T12:37:33.593Z 2019-04-16 12:37:35,594 [INFO] [TezChild] |hive.TimestampUDF|: endTime: 2019-04-16T12:37:35.594Z 2019-04-16 12:37:35,596 [INFO] [TezChild] |hive.TimestampUDF|: duration: 2001 2019-04-16 12:37:35,597 [INFO] [TezChild] |hive.TimestampUDF|: == TimestampUDF == 2019-04-16 12:37:35,597 [INFO] [TezChild] |hive.TimestampUDF|: startTime: 2019-04-16T12:37:35.597Z 2019-04-16 12:37:37,597 [INFO] [TezChild] |hive.TimestampUDF|: endTime: 2019-04-16T12:37:37.597Z 2019-04-16 12:37:37,597 [INFO] [TezChild] |hive.TimestampUDF|: duration: 2000 2019-04-16 12:37:37,598 [INFO] [TezChild] |hive.TimestampUDF|: == TimestampUDF == 2019-04-16 12:37:37,598 [INFO] [TezChild] |hive.TimestampUDF|: startTime: 2019-04-16T12:37:37.598Z 2019-04-16 12:37:39,598 [INFO] [TezChild] |hive.TimestampUDF|: endTime: 2019-04-16T12:37:39.598Z 2019-04-16 12:37:39,598 [INFO] [TezChild] |hive.TimestampUDF|: duration: 2000 2019-04-16 12:37:39,603 [INFO] [TezChild] |exec.FileSinkOperator|: Final Path: FS hdfs://ip-172-31-69-167.ec2.internal:8020/tmp/hive/hive/4adc7ed2-01cc-481b-b2aa-bbe41551ac08/hive_2019-04-16_12-37-02_586_6615575765504587239-1/-mr-10001/.hive-staging_hive_2019-04-16_12-37-02_586_6615575765504587239-1/_tmp.-ext-10002/000000_0 2019-04-16 12:37:39,603 [INFO] [TezChild] |exec.FileSinkOperator|: Writing to temp file: FS hdfs://ip-172-31-69-167.ec2.internal:8020/tmp/hive/hive/4adc7ed2-01cc-481b-b2aa-bbe41551ac08/hive_2019-04-16_12-37-02_586_6615575765504587239-1/-mr-10001/.hive-staging_hive_2019-04-16_12-37-02_586_6615575765504587239-1/_task_tmp.-ext-10002/_tmp.000000_0 2019-04-16 12:37:39,603 [INFO] [TezChild] |exec.FileSinkOperator|: New Final Path: FS hdfs://ip-172-31-69-167.ec2.internal:8020/tmp/hive/hive/4adc7ed2-01cc-481b-b2aa-bbe41551ac08/hive_2019-04-16_12-37-02_586_6615575765504587239-1/-mr-10001/.hive-staging_hive_2019-04-16_12-37-02_586_6615575765504587239-1/_tmp.-ext-10002/000000_0 2019-04-16 12:37:39,645 [INFO] [TezChild] |exec.FileSinkOperator|: FS[4]: records written - 1 2019-04-16 12:37:39,645 [INFO] [TezChild] |tez.ObjectCache|: Releasing key: hive_20190416123702_5ee0ce4b-526b-4828-b572-3bb9507e0f85_Map 1__MAP_PLAN__ 2019-04-16 12:37:39,645 [INFO] [TezChild] |tez.ObjectCache|: Releasing key: hive_20190416123702_5ee0ce4b-526b-4828-b572-3bb9507e0f85_DynamicValueRegistry 2019-04-16 12:37:39,645 [INFO] [TezChild] |vector.VectorMapOperator|: RECORDS_OUT_INTERMEDIATE_Map_1:0, DESERIALIZE_ERRORS:0, RECORDS_IN_Map_1:1, RECORDS_OUT_OPERATOR_MAP_0:0, 2019-04-16 12:37:39,645 [INFO] [TezChild] |exec.TableScanOperator|: RECORDS_OUT_INTERMEDIATE_Map_1:0, RECORDS_OUT_OPERATOR_TS_0:1, 2019-04-16 12:37:39,645 [INFO] [TezChild] |vector.VectorSelectOperator|: RECORDS_OUT_OPERATOR_SEL_3:1, RECORDS_OUT_INTERMEDIATE_Map_1:0, 2019-04-16 12:37:39,645 [INFO] [TezChild] |exec.FileSinkOperator|: FS[4]: records written - 1 2019-04-16 12:37:39,702 [INFO] [TezChild] |vector.VectorFileSinkOperator|: RECORDS_OUT_INTERMEDIATE_Map_1:0, RECORDS_OUT_OPERATOR_FS_4:1, RECORDS_OUT_0:1, 2019-04-16 12:37:39,702 [INFO] [TezChild] |task.TaskRunner2Callable|: Closing task, taskAttemptId=attempt_1555415227039_0004_1_00_000000_0 2019-04-16 12:37:39,702 [INFO] [TezChild] |output.MROutput|: out_Map 1 closed 2019-04-16 12:37:39,703 [INFO] [TezChild] |runtime.LogicalIOProcessorRuntimeTask|: Joining on EventRouter 2019-04-16 12:37:39,705 [INFO] [TezChild] |task.TaskRunner2Callable|: Task completed, taskAttemptId=attempt_1555415227039_0004_1_00_000000_0, askedToStop=false 2019-04-16 12:37:39,705 [INFO] [TezChild] |task.TaskRunner2Callable|: Cleaning up task attempt_1555415227039_0004_1_00_000000_0, stopRequested=false 2019-04-16 12:37:39,706 [INFO] [TezChild] |runtime.LogicalIOProcessorRuntimeTask|: Final Counters for attempt_1555415227039_0004_1_00_000000_0: Counters: 28 [[File System Counters HDFS_BYTES_READ=712, HDFS_BYTES_WRITTEN=154, HDFS_READ_OPS=5, HDFS_WRITE_OPS=2, HDFS_OP_CREATE=1, HDFS_OP_GET_FILE_STATUS=3, HDFS_OP_OPEN=2, HDFS_OP_RENAME=1][org.apache.tez.common.counters.TaskCounter GC_TIME_MILLIS=79, CPU_MILLISECONDS=3480, PHYSICAL_MEMORY_BYTES=253755392, VIRTUAL_MEMORY_BYTES=5422989312, COMMITTED_HEAP_BYTES=253755392, INPUT_RECORDS_PROCESSED=1, INPUT_SPLIT_LENGTH_BYTES=276, OUTPUT_RECORDS=0][HIVE CREATED_FILES=1, DESERIALIZE_ERRORS=0, RECORDS_IN_Map_1=1, RECORDS_OUT_0=1, RECORDS_OUT_INTERMEDIATE_Map_1=0, RECORDS_OUT_OPERATOR_FS_4=1, RECORDS_OUT_OPERATOR_MAP_0=0, RECORDS_OUT_OPERATOR_SEL_3=1, RECORDS_OUT_OPERATOR_TS_0=1][TaskCounter_Map_1_INPUT_idtable INPUT_RECORDS_PROCESSED=1, INPUT_SPLIT_LENGTH_BYTES=276][TaskCounter_Map_1_OUTPUT_out_Map_1 OUTPUT_RECORDS=0]] 2019-04-16 12:37:39,713 [INFO] [main] |task.TezTaskRunner2|: TaskRunnerResult for attempt_1555415227039_0004_1_00_000000_0 : TaskRunner2Result{endReason=SUCCESS, containerShutdownRequested=false} 2019-04-16 12:37:39,713 [INFO] [main] |task.TezChild|: TaskRunner2Result: TaskRunner2Result{endReason=SUCCESS, containerShutdownRequested=false} 2019-04-16 12:37:39,814 [INFO] [main] |common.TezUtilsInternal|: Redirecting log file based on addend: End of LogType:syslog_attempt_1555415227039_0004_1_00_000000_0 ****************************************************************************************************************