diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index aa27a01174..f74485bae5 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2163,6 +2163,17 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "the Tez UI representing the work that was done. Used by Spark to set the query name, will show up in the\n" + "Spark UI."), + SYSLOG_INPUT_FORMAT_FILE_PRUNING("hive.syslog.input.format.file.pruning", true, + "Whether syslog input format should prune files based on timestamp (ts) column in sys.logs table."), + SYSLOG_INPUT_FORMAT_FILE_TIME_SLICE("hive.syslog.input.format.file.time.slice", "300s", + new TimeValidator(TimeUnit.SECONDS, 0L, false, Long.MAX_VALUE, false), + "Files stored in sys.logs typically are chunked with time interval. For example: depending on the\n" + + "logging library used this represents the flush interval/time slice. \n" + + "If time slice/flust interval is set to 5 minutes, then the expectation is that the filename \n" + + "2019-01-02-10-00_0.log represent time range from 10:00 to 10:05.\n" + + "This time slice should align with the flush interval of the logging library else file pruning may\n" + + "incorrectly prune files leading to incorrect results from sys.logs table."), + HIVEOPTIMIZEBUCKETINGSORTING("hive.optimize.bucketingsorting", true, "Don't create a reducer for enforcing \n" + "bucketing/sorting for queries of the form: \n" + diff --git a/data/files/syslog-hs2-2.log b/data/files/syslog-hs2-2.log new file mode 100644 index 0000000000..da9bb3b8ee --- /dev/null +++ b/data/files/syslog-hs2-2.log @@ -0,0 +1,331 @@ +Connection to metastore-service.warehouse-1553216530-7m7s.svc.cluster.local 9083 port [tcp/*] succeeded! ++ DOWNLOAD_PATH=/custom-jars ++ [[ -z '' ]] ++ echo 'CUSTOM_JARS_URL is not defined. Skipping jar download from url..' +CUSTOM_JARS_URL is not defined. Skipping jar download from url.. ++ [[ -z s3a://edws-llap-perf/custom-jars/prasanth-compute ]] ++ cd /custom-jars ++ /usr/hdp/3.0.100.0-383/hadoop/bin/hadoop s3guard prune -seconds 1 s3a://edws-llap-perf/custom-jars/prasanth-compute/ +ERROR: Tools helper /usr/hdp/3.0.100.0-383/hadoop/libexec/tools/hadoop-aws.sh was not found. +log4j:WARN No appenders could be found for logger (org.apache.hadoop.util.Shell). +log4j:WARN Please initialize the log4j system properly. +log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info. +CUSTOM_JARS_PATH is set to s3a://edws-llap-perf/custom-jars/prasanth-compute. Copying jar/tar to /custom-jars.. ++ echo 'CUSTOM_JARS_PATH is set to s3a://edws-llap-perf/custom-jars/prasanth-compute. Copying jar/tar to /custom-jars..' ++ /usr/hdp/3.0.100.0-383/hadoop/bin/hdfs dfs -copyToLocal 's3a://edws-llap-perf/custom-jars/prasanth-compute/*' . +log4j:WARN No appenders could be found for logger (org.apache.hadoop.util.Shell). +log4j:WARN Please initialize the log4j system properly. +log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info. ++ '[' 0 == 0 ']' ++ ls -l . +total 52800 +-rw-r--r-- 1 hive hive 42092027 Mar 22 01:06 hive-exec-3.1.0-SNAPSHOT.jar +-rw-r--r-- 1 hive hive 11970900 Mar 22 01:06 hive-standalone-metastore-3.1.0-SNAPSHOT.jar ++ '[' -d ./lib ']' ++ cd /custom-jars ++ find . -type f -iname '*.tar.gz' -print0 -execdir tar xf '{}' ';' -delete ++ find . -name '*.jar' -exec md5sum '{}' ';' +29adaafae170426713fc75a63508c224 ./hive-exec-3.1.0-SNAPSHOT.jar +b04082bd54ec9726876b2100ef856d86 ./hive-standalone-metastore-3.1.0-SNAPSHOT.jar ++ . /fluentd-utils.sh +++ set -x +++ FLUENTD_CONF=/var/tmp/fluentd.conf +++ FLUENTD_PIDFILE=/var/tmp/fluentd.pid ++ start_fluentd ++ '[' '' '!=' yarn ']' ++ return ++ '[' hiveserver2 == query-executor ']' ++ '[' hiveserver2 == query-coordinator ']' ++ '[' hiveserver2 == data-load ']' ++ exec /hive-entrypoint.sh +++ /usr/hdp/3.0.100.0-383/hadoop/bin/hadoop classpath ++ SCHEMATOOL_COMMAND='/usr/hdp/3.0.100.0-383/hive/bin/schematool -dbType postgres' ++ '[' hiveserver2 == metastore ']' ++ [[ hiveserver2 == \m\e\t\a\s\t\o\r\e ]] ++ [[ hiveserver2 == \h\i\v\e\s\e\r\v\e\r\2 ]] ++ export 'HADOOP_CLIENT_OPTS=-Djdk.tls.disabledAlgorithms=SSLv3,GCM -Xms8192M -Xmx13107M -Xloggc:/var/log/hive/hiveserver2-gc-%t.log -XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCCause -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M -javaagent:/jmx-exporter/jmx_prometheus_javaagent-0.11.0.jar=35000:/jmx-exporter/config.yml -agentlib:jdwp=transport=dt_socket,address=50505,suspend=n,server=y' ++ HADOOP_CLIENT_OPTS='-Djdk.tls.disabledAlgorithms=SSLv3,GCM -Xms8192M -Xmx13107M -Xloggc:/var/log/hive/hiveserver2-gc-%t.log -XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCCause -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M -javaagent:/jmx-exporter/jmx_prometheus_javaagent-0.11.0.jar=35000:/jmx-exporter/config.yml -agentlib:jdwp=transport=dt_socket,address=50505,suspend=n,server=y' ++ exec /usr/hdp/3.0.100.0-383/hive/bin/hive --skiphadoopversion --skiphbasecp --service hiveserver2 +2019-03-22 01:06:11: Starting HiveServer2 +Listening for transport dt_socket at address: 50505 +SLF4J: Class path contains multiple SLF4J bindings. +SLF4J: Found binding in [jar:file:/usr/hdp/3.0.100.0-383/hive/lib/log4j-slf4j-impl-2.10.0.jar!/org/slf4j/impl/StaticLoggerBinder.class] +SLF4J: Found binding in [jar:file:/usr/hdp/3.0.100.0-383/hadoop/lib/slf4j-log4j12-1.7.25.jar!/org/slf4j/impl/StaticLoggerBinder.class] +SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation. +SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory] +<14>1 2019-03-22T01:08:36.330Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Starting task [Stage-1:MAPRED] in serial mode +<14>1 2019-03-22T01:08:36.345Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Context" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] New scratch dir is s3a://edws-llap-perf/tmp/hive/hive/fd9967ca-be37-4e5f-b291-0167792fd42c/hive_2019-03-22_01-08-35_261_5931730823725478551-8 +<15>1 2019-03-22T01:08:36.345Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<14>1 2019-03-22T01:08:36.345Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezSessionPoolManager" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Choosing a session from the defaultQueuePool +<15>1 2019-03-22T01:08:36.345Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<14>1 2019-03-22T01:08:36.345Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Subscribed to counters: [] for queryId: hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0 +<14>1 2019-03-22T01:08:36.345Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Session is already open +<15>1 2019-03-22T01:08:36.345Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<14>1 2019-03-22T01:08:36.345Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Dag name: insert overwrite tabl...rnal.planes_external (Stage-1) +<15>1 2019-03-22T01:08:36.346Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<15>1 2019-03-22T01:08:36.346Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<14>1 2019-03-22T01:08:36.346Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="exec.SerializationUtilities" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Serializing ReduceWork using kryo +<15>1 2019-03-22T01:08:36.348Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<14>1 2019-03-22T01:08:36.348Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="exec.Utilities" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Serialized plan (via RPC) - name: Reducer 2 size: 2.70KB +<14>1 2019-03-22T01:08:37.126Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="fs.FSStatsPublisher" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] created : s3a://edws-llap-perf/tmp/hive/hive/fd9967ca-be37-4e5f-b291-0167792fd42c/hive_2019-03-22_01-08-35_903_8908558446822068563-8/-mr-10002 +<15>1 2019-03-22T01:08:37.130Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<15>1 2019-03-22T01:08:37.130Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<14>1 2019-03-22T01:08:37.131Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Context" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] New scratch dir is s3a://edws-llap-perf/tmp/hive/hive/fd9967ca-be37-4e5f-b291-0167792fd42c/hive_2019-03-22_01-08-35_261_5931730823725478551-8 +<14>1 2019-03-22T01:08:37.408Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.DagUtils" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Vertex has custom input? false +<15>1 2019-03-22T01:08:37.408Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<14>1 2019-03-22T01:08:37.408Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="exec.SerializationUtilities" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Serializing MapWork using kryo +<15>1 2019-03-22T01:08:37.410Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<14>1 2019-03-22T01:08:37.410Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="exec.Utilities" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Serialized plan (via RPC) - name: Map 1 size: 5.30KB +<14>1 2019-03-22T01:08:37.614Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="fs.FSStatsPublisher" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] created : s3a://edws-llap-perf/tmp/hive/hive/fd9967ca-be37-4e5f-b291-0167792fd42c/hive_2019-03-22_01-08-35_261_5931730823725478551-8/-mr-10001 +<15>1 2019-03-22T01:08:37.615Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<15>1 2019-03-22T01:08:37.620Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<15>1 2019-03-22T01:08:37.620Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<14>1 2019-03-22T01:08:37.620Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="client.TezClient" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Submitting dag to TezSession, sessionName=HIVE-183d8fda-a26c-4308-b7f1-ac55645b71dd, applicationId=application_1553216658672_0000, dagName=insert overwrite tabl...rnal.planes_external (Stage-1), callerContext={ context=HIVE, callerType=HIVE_QUERY_ID, callerId=hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0 } +<14>1 2019-03-22T01:08:37.620Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="common.TezYARNUtils" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Ignoring 'tez.lib.uris' since 'tez.ignore.lib.uris' is set to true +<14>1 2019-03-22T01:08:37.620Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="common.TezYARNUtils" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Ignoring 'tez.lib.uris' since 'tez.ignore.lib.uris' is set to true +<14>1 2019-03-22T01:08:37.722Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="client.TezClient" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Submitted dag to TezSession, sessionName=HIVE-183d8fda-a26c-4308-b7f1-ac55645b71dd, applicationId=application_1553216658672_0000, dagId=dag_1553216658672_0000_4, dagName=insert overwrite tabl...rnal.planes_external (Stage-1) +<15>1 2019-03-22T01:08:37.722Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<15>1 2019-03-22T01:08:37.722Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<15>1 2019-03-22T01:08:37.722Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +Status: Running (Executing on YARN cluster with App id application_1553216658672_0000) + +<15>1 2019-03-22T01:08:38.236Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<14>1 2019-03-22T01:08:38.236Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Status: Running (Executing on YARN cluster with App id application_1553216658672_0000) +<15>1 2019-03-22T01:08:38.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<14>1 2019-03-22T01:08:38.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="monitoring.RenderStrategy$LogToFileFunction" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Map 1: 0(+1)/1 Reducer 2: 0/1 +<14>1 2019-03-22T01:08:39.843Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-34"] Using the default value passed in for log id: fd9967ca-be37-4e5f-b291-0167792fd42c +<14>1 2019-03-22T01:08:39.843Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-34"] Updating thread name to fd9967ca-be37-4e5f-b291-0167792fd42c HiveServer2-HttpHandler-Pool: Thread-34 +<14>1 2019-03-22T01:08:39.843Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="fd9967ca-be37-4e5f-b291-0167792fd42c HiveServer2-HttpHandler-Pool: Thread-34"] Using the default value passed in for log id: fd9967ca-be37-4e5f-b291-0167792fd42c +<14>1 2019-03-22T01:08:39.843Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="fd9967ca-be37-4e5f-b291-0167792fd42c HiveServer2-HttpHandler-Pool: Thread-34"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-34 +<15>1 2019-03-22T01:08:41.273Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<15>1 2019-03-22T01:08:41.273Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<14>1 2019-03-22T01:08:41.273Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="monitoring.RenderStrategy$LogToFileFunction" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Map 1: 1/1 Reducer 2: 0(+1)/1 +<15>1 2019-03-22T01:08:42.226Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +Status: DAG finished successfully in 3.99 seconds + +<14>1 2019-03-22T01:08:42.226Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="monitoring.RenderStrategy$LogToFileFunction" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Map 1: 1/1 Reducer 2: 1/1 +Query Execution Summary +<15>1 2019-03-22T01:08:42.226Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +---------------------------------------------------------------------------------------------- +OPERATION DURATION<14>1 2019-03-22T01:08:42.226Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Status: DAG finished successfully in 3.99 seconds + +---------------------------------------------------------------------------------------------- +Compile Query 1.05s +<14>1 2019-03-22T01:08:42.226Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +Prepare Plan 1.31s +<14>1 2019-03-22T01:08:42.226Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Query Execution Summary +Get Query Coordinator (AM) 0.00s +Submit Plan 0.10s +<14>1 2019-03-22T01:08:42.226Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] ---------------------------------------------------------------------------------------------- +Start DAG 0.51s +Run DAG 3.99s<14>1 2019-03-22T01:08:42.226Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] OPERATION DURATION + +---------------------------------------------------------------------------------------------- + +<14>1 2019-03-22T01:08:42.226Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] ---------------------------------------------------------------------------------------------- +<14>1 2019-03-22T01:08:42.226Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Compile Query 1.05s +<14>1 2019-03-22T01:08:42.226Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Prepare Plan 1.31s +<14>1 2019-03-22T01:08:42.227Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Get Query Coordinator (AM) 0.00s +<14>1 2019-03-22T01:08:42.227Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Submit Plan 0.10s +<14>1 2019-03-22T01:08:42.227Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Start DAG 0.51s +<14>1 2019-03-22T01:08:42.227Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Run DAG 3.99s +<14>1 2019-03-22T01:08:42.227Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] ---------------------------------------------------------------------------------------------- +<14>1 2019-03-22T01:08:42.227Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +Task Execution Summary +---------------------------------------------------------------------------------------------- + VERTICES DURATION(ms) CPU_TIME(ms) GC_TIME(ms) INPUT_RECORDS OUTPUT_RECORDS +---------------------------------------------------------------------------------------------- +<14>1 2019-03-22T01:08:42.229Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Task Execution Summary +<14>1 2019-03-22T01:08:42.229Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] ---------------------------------------------------------------------------------------------- +<14>1 2019-03-22T01:08:42.229Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] VERTICES DURATION(ms) CPU_TIME(ms) GC_TIME(ms) INPUT_RECORDS OUTPUT_RECORDS +<14>1 2019-03-22T01:08:42.229Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] ---------------------------------------------------------------------------------------------- + Map 1 3036.00 0 0 5,029 1 +<14>1 2019-03-22T01:08:42.230Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Map 1 3036.00 0 0 5,029 1 + Reducer 2 953.00 0 0 1 0 +---------------------------------------------------------------------------------------------- + +LLAP IO Summary +<14>1 2019-03-22T01:08:42.232Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Reducer 2 953.00 0 0 1 0 +<14>1 2019-03-22T01:08:42.232Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] ---------------------------------------------------------------------------------------------- +<14>1 2019-03-22T01:08:42.232Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<14>1 2019-03-22T01:08:42.232Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] LLAP IO Summary +---------------------------------------------------------------------------------------------- + VERTICES ROWGROUPS META_HIT META_MISS DATA_HIT DATA_MISS TOTAL_IO +---------------------------------------------------------------------------------------------- +<14>1 2019-03-22T01:08:42.233Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] ---------------------------------------------------------------------------------------------- + Map 1 1 0 2 0B 37.78KB 0.15s +---------------------------------------------------------------------------------------------- + +FileSystem Counters Summary<14>1 2019-03-22T01:08:42.233Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] VERTICES ROWGROUPS META_HIT META_MISS DATA_HIT DATA_MISS TOTAL_IO + +<14>1 2019-03-22T01:08:42.233Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] ---------------------------------------------------------------------------------------------- + +Scheme: FILE +----------------------------------------------------------------------------------------------<14>1 2019-03-22T01:08:42.233Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Map 1 1 0 2 0B 37.78KB 0.15s + + VERTICES BYTES_READ READ_OPS LARGE_READ_OPS BYTES_WRITTEN WRITE_OPS +---------------------------------------------------------------------------------------------- +<14>1 2019-03-22T01:08:42.233Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] ---------------------------------------------------------------------------------------------- +<14>1 2019-03-22T01:08:42.233Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<14>1 2019-03-22T01:08:42.233Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] FileSystem Counters Summary +<14>1 2019-03-22T01:08:42.233Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<14>1 2019-03-22T01:08:42.233Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Scheme: FILE +<14>1 2019-03-22T01:08:42.233Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] ---------------------------------------------------------------------------------------------- +<14>1 2019-03-22T01:08:42.233Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] VERTICES BYTES_READ READ_OPS LARGE_READ_OPS BYTES_WRITTEN WRITE_OPS +<14>1 2019-03-22T01:08:42.233Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] ---------------------------------------------------------------------------------------------- + Map 1 0B 0 0 1.99KB 0 +<14>1 2019-03-22T01:08:42.234Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Map 1 0B 0 0 1.99KB 0 + Reducer 2 0B 0 0 0B 0 +---------------------------------------------------------------------------------------------- + +Scheme: S3A +---------------------------------------------------------------------------------------------- + VERTICES BYTES_READ READ_OPS LARGE_READ_OPS BYTES_WRITTEN WRITE_OPS +----------------------------------------------------------------------------------------------<14>1 2019-03-22T01:08:42.235Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Reducer 2 0B 0 0 0B 0 + +<14>1 2019-03-22T01:08:42.235Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] ---------------------------------------------------------------------------------------------- +<14>1 2019-03-22T01:08:42.235Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<14>1 2019-03-22T01:08:42.235Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Scheme: S3A +<14>1 2019-03-22T01:08:42.235Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] ---------------------------------------------------------------------------------------------- +<14>1 2019-03-22T01:08:42.235Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] VERTICES BYTES_READ READ_OPS LARGE_READ_OPS BYTES_WRITTEN WRITE_OPS +<14>1 2019-03-22T01:08:42.235Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] ---------------------------------------------------------------------------------------------- + Map 1 54.38KB 11 0 0B 8 +<14>1 2019-03-22T01:08:42.236Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Map 1 54.38KB 11 0 0B 8 + Reducer 2 0B 6 0 0B 7 +---------------------------------------------------------------------------------------------- + +<14>1 2019-03-22T01:08:42.236Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Reducer 2 0B 6 0 0B 7 +<14>1 2019-03-22T01:08:42.236Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] ---------------------------------------------------------------------------------------------- +<14>1 2019-03-22T01:08:42.236Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezSessionPoolManager" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] The session 183d8fda-a26c-4308-b7f1-ac55645b71dd belongs to the pool. Put it back in +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] org.apache.tez.common.counters.DAGCounter: +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] NUM_SUCCEEDED_TASKS: 2 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] TOTAL_LAUNCHED_TASKS: 2 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] DATA_LOCAL_TASKS: 1 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] AM_CPU_MILLISECONDS: 580 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] AM_GC_TIME_MILLIS: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] File System Counters: +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] FILE_BYTES_READ: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] FILE_BYTES_WRITTEN: 1985 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] FILE_READ_OPS: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] FILE_LARGE_READ_OPS: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] FILE_WRITE_OPS: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] S3A_BYTES_READ: 54377 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] S3A_BYTES_WRITTEN: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] S3A_READ_OPS: 17 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] S3A_LARGE_READ_OPS: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] S3A_WRITE_OPS: 15 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] org.apache.tez.common.counters.TaskCounter: +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] SPILLED_RECORDS: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] NUM_SHUFFLED_INPUTS: 1 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] NUM_FAILED_SHUFFLE_INPUTS: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] TASK_DURATION_MILLIS: 4179 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] INPUT_RECORDS_PROCESSED: 5030 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] INPUT_SPLIT_LENGTH_BYTES: 38746 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] OUTPUT_RECORDS: 1 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] OUTPUT_LARGE_RECORDS: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] OUTPUT_BYTES: 3982 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] OUTPUT_BYTES_WITH_OVERHEAD: 3992 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] OUTPUT_BYTES_PHYSICAL: 1977 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] ADDITIONAL_SPILLS_BYTES_WRITTEN: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] ADDITIONAL_SPILLS_BYTES_READ: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] ADDITIONAL_SPILL_COUNT: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] SHUFFLE_BYTES: 1953 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] SHUFFLE_BYTES_DECOMPRESSED: 3992 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] SHUFFLE_BYTES_TO_MEM: 1953 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] SHUFFLE_BYTES_TO_DISK: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] SHUFFLE_BYTES_DISK_DIRECT: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] SHUFFLE_PHASE_TIME: 93 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] FIRST_EVENT_RECEIVED: 11 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] LAST_EVENT_RECEIVED: 11 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] HIVE: +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] CREATED_FILES: 2 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] DESERIALIZE_ERRORS: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] RECORDS_IN_Map_1: 5029 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] RECORDS_OUT_0: 1 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] RECORDS_OUT_1_airline_ontime.planes: 5029 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] RECORDS_OUT_INTERMEDIATE_Map_1: 1 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] RECORDS_OUT_OPERATOR_FS_2: 5029 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] RECORDS_OUT_OPERATOR_FS_9: 1 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] RECORDS_OUT_OPERATOR_GBY_5: 1 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] RECORDS_OUT_OPERATOR_GBY_7: 1 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] RECORDS_OUT_OPERATOR_MAP_0: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] RECORDS_OUT_OPERATOR_RS_6: 1 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] RECORDS_OUT_OPERATOR_SEL_1: 5029 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] RECORDS_OUT_OPERATOR_SEL_4: 5029 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] RECORDS_OUT_OPERATOR_TS_0: 5029 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] TaskCounter_Map_1_INPUT_planes_external: +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] INPUT_RECORDS_PROCESSED: 5029 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] INPUT_SPLIT_LENGTH_BYTES: 38746 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] TaskCounter_Map_1_OUTPUT_Reducer_2: +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] ADDITIONAL_SPILLS_BYTES_READ: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] ADDITIONAL_SPILLS_BYTES_WRITTEN: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] ADDITIONAL_SPILL_COUNT: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] OUTPUT_BYTES: 3982 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] OUTPUT_BYTES_PHYSICAL: 1977 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] OUTPUT_BYTES_WITH_OVERHEAD: 3992 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] OUTPUT_LARGE_RECORDS: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] OUTPUT_RECORDS: 1 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] SPILLED_RECORDS: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] TaskCounter_Reducer_2_INPUT_Map_1: +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] FIRST_EVENT_RECEIVED: 11 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] INPUT_RECORDS_PROCESSED: 1 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] LAST_EVENT_RECEIVED: 11 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] NUM_FAILED_SHUFFLE_INPUTS: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] NUM_SHUFFLED_INPUTS: 1 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] SHUFFLE_BYTES: 1953 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] SHUFFLE_BYTES_DECOMPRESSED: 3992 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] SHUFFLE_BYTES_DISK_DIRECT: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] SHUFFLE_BYTES_TO_DISK: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] SHUFFLE_BYTES_TO_MEM: 1953 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] SHUFFLE_PHASE_TIME: 93 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] TaskCounter_Reducer_2_OUTPUT_out_Reducer_2: +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] OUTPUT_RECORDS: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] org.apache.hadoop.hive.llap.counters.LlapIOCounters: +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] ALLOCATED_BYTES: 11010048 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] ALLOCATED_USED_BYTES: 88690 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] CACHE_HIT_BYTES: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] CACHE_MISS_BYTES: 37775 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] CONSUMER_TIME_NS: 816530216 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] DECODE_TIME_NS: 29308516 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] HDFS_TIME_NS: 401525272 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] METADATA_CACHE_MISS: 2 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] NUM_DECODED_BATCHES: 1 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] NUM_VECTOR_BATCHES: 5 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] ROWS_EMITTED: 5029 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] SELECTED_ROWGROUPS: 1 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] TOTAL_IO_TIME_NS: 145249644 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] org.apache.hadoop.hive.llap.counters.LlapWmCounters: +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] GUARANTEED_QUEUED_NS: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] GUARANTEED_RUNNING_NS: 0 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] SPECULATIVE_QUEUED_NS: 208330 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] SPECULATIVE_RUNNING_NS: 4166761931 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] org.apache.hadoop.hive.ql.exec.tez.HiveInputCounters: +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] GROUPED_INPUT_SPLITS_Map_1: 1 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] INPUT_DIRECTORIES_Map_1: 1 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] INPUT_FILES_Map_1: 1 +<14>1 2019-03-22T01:08:42.237Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] RAW_INPUT_SPLITS_Map_1: 1 +<15>1 2019-03-22T01:08:42.680Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<15>1 2019-03-22T01:08:42.680Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<15>1 2019-03-22T01:08:42.680Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<15>1 2019-03-22T01:08:43.031Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<15>1 2019-03-22T01:08:43.388Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<15>1 2019-03-22T01:08:43.388Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<15>1 2019-03-22T01:08:43.388Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<15>1 2019-03-22T01:08:43.689Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] +<14>1 2019-03-22T01:08:43.797Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Starting task [Stage-2:DEPENDENCY_COLLECTION] in serial mode +<14>1 2019-03-22T01:08:43.797Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Starting task [Stage-0:MOVE] in serial mode +Loading data to table airline_ontime.planes +<14>1 2019-03-22T01:08:43.797Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="exec.Task" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Loading data to table airline_ontime.planes from s3a://edws-llap-perf/clusters/cluster-1547937522-w74v/warehouse-1553216530-7m7s/warehouse/tablespace/managed/hive/airline_ontime.db/planes/.hive-staging_hive_2019-03-22_01-08-35_261_5931730823725478551-8/-ext-10000 +<14>1 2019-03-22T01:08:44.668Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metadata.Hive" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Moving bucket s3a://edws-llap-perf/clusters/cluster-1547937522-w74v/warehouse-1553216530-7m7s/warehouse/tablespace/managed/hive/airline_ontime.db/planes/.hive-staging_hive_2019-03-22_01-08-35_261_5931730823725478551-8/-ext-10000/000000_0/base_0000001/bucket_00000 to s3a://edws-llap-perf/clusters/cluster-1547937522-w74v/warehouse-1553216530-7m7s/warehouse/tablespace/managed/hive/airline_ontime.db/planes/base_0000001/bucket_00000 +<14>1 2019-03-22T01:08:45.497Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Starting task [Stage-3:STATS] in serial mode +<14>1 2019-03-22T01:08:45.498Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="stats.BasicStatsTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Executing stats task +<14>1 2019-03-22T01:08:45.503Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="fs.FSStatsPublisher" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] created : s3a://edws-llap-perf/tmp/hive/hive/fd9967ca-be37-4e5f-b291-0167792fd42c/hive_2019-03-22_01-08-35_261_5931730823725478551-8/-mr-10001 +<14>1 2019-03-22T01:08:45.805Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="FileOperations" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Read stats for : airline_ontime.planes/ numRows 5029 +<14>1 2019-03-22T01:08:45.805Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="FileOperations" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Read stats for : airline_ontime.planes/ rawDataSize 0 +Table airline_ontime.planes stats: [numFiles=1, numRows=5029, totalSize=39362, rawDataSize=0] +<14>1 2019-03-22T01:08:45.941Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="stats.BasicStatsTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Table airline_ontime.planes stats: [numFiles=1, numRows=5029, totalSize=39362, rawDataSize=0] +<14>1 2019-03-22T01:08:45.941Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="stats.BasicStatsTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Table airline_ontime.planes stats: [numFiles=1, numRows=5029, totalSize=39362, rawDataSize=0] +<14>1 2019-03-22T01:08:46.294Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="mapred.FileInputFormat" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] Total input files to process : 1 +<15>1 2019-03-22T01:08:46.392Z hiveserver2-59d448c784-c4fxj hiveserver2 1 38f41977-4c3e-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190322010835_300afc89-ce66-4f50-afe4-e34f2f32b7d0" sessionId="fd9967ca-be37-4e5f-b291-0167792fd42c" thread="HiveServer2-Background-Pool: Thread-209"] diff --git a/data/files/syslog-hs2.log b/data/files/syslog-hs2.log new file mode 100644 index 0000000000..ab25d62387 --- /dev/null +++ b/data/files/syslog-hs2.log @@ -0,0 +1,756 @@ ++ . /fluentd-utils.sh +++ set -x +++ FLUENTD_CONF=/var/tmp/fluentd.conf +++ FLUENTD_PIDFILE=/var/tmp/fluentd.pid ++ start_fluentd ++ '[' '' '!=' yarn ']' ++ return ++ '[' hiveserver2 == query-executor ']' ++ '[' hiveserver2 == query-coordinator ']' ++ '[' hiveserver2 == data-load ']' ++ exec /hive-entrypoint.sh ++ export 'HADOOP_CLASSPATH=/usr/hdp/3.0.100.0-373/hadoop/conf:/usr/hdp/3.0.100.0-373/hadoop/lib/*:/usr/hdp/3.0.100.0-373/hadoop/.//*:/usr/hdp/3.0.100.0-373/hadoop-hdfs/./:/usr/hdp/3.0.100.0-373/hadoop-hdfs/lib/*:/usr/hdp/3.0.100.0-373/hadoop-hdfs/.//*:/usr/hdp/3.0.100.0-373/hadoop-mapreduce/lib/*:/usr/hdp/3.0.100.0-373/hadoop-mapreduce/.//*:/usr/hdp/3.0.100.0-373/hadoop-yarn/./:/usr/hdp/3.0.100.0-373/hadoop-yarn/lib/*:/usr/hdp/3.0.100.0-373/hadoop-yarn/.//*:/usr/hdp/3.0.100.0-373/tez/*:/usr/hdp/3.0.100.0-373/tez/lib/*:/usr/hdp/3.0.100.0-373/tez/conf:/usr/hdp/3.0.100.0-373/tez/hadoop-shim-0.9.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/hadoop-shim-2.8-0.9.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/lib:/usr/hdp/3.0.100.0-373/tez/tez-api-0.9.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/tez-common-0.9.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/tez-dag-0.9.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/tez-examples-0.9.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/tez-history-parser-0.9.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/tez-javadoc-tools-0.9.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/tez-job-analyzer-0.9.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/tez-mapreduce-0.9.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/tez-protobuf-history-plugin-0.9.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/tez-runtime-internals-0.9.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/tez-runtime-library-0.9.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/tez-tests-0.9.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/tez-yarn-timeline-cache-plugin-0.9.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/tez-yarn-timeline-history-0.9.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/tez-yarn-timeline-history-with-acls-0.9.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/tez-yarn-timeline-history-with-fs-0.9.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/ui:/usr/hdp/3.0.100.0-373/tez/lib/RoaringBitmap-0.4.9.jar:/usr/hdp/3.0.100.0-373/tez/lib/async-http-client-1.9.40.jar:/usr/hdp/3.0.100.0-373/tez/lib/commons-cli-1.2.jar:/usr/hdp/3.0.100.0-373/tez/lib/commons-codec-1.4.jar:/usr/hdp/3.0.100.0-373/tez/lib/commons-collections-3.2.2.jar:/usr/hdp/3.0.100.0-373/tez/lib/commons-collections4-4.1.jar:/usr/hdp/3.0.100.0-373/tez/lib/commons-io-2.4.jar:/usr/hdp/3.0.100.0-373/tez/lib/commons-lang-2.6.jar:/usr/hdp/3.0.100.0-373/tez/lib/commons-math3-3.1.1.jar:/usr/hdp/3.0.100.0-373/tez/lib/gcs-connector-1.9.10.3.0.100.0-373-shaded.jar:/usr/hdp/3.0.100.0-373/tez/lib/guava-11.0.2.jar:/usr/hdp/3.0.100.0-373/tez/lib/hadoop-aws-3.1.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/lib/hadoop-azure-3.1.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/lib/hadoop-azure-datalake-3.1.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/lib/hadoop-hdfs-client-3.1.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/lib/hadoop-mapreduce-client-common-3.1.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/lib/hadoop-mapreduce-client-core-3.1.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/lib/hadoop-yarn-registry-3.1.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/lib/hadoop-yarn-server-timeline-pluginstorage-3.1.1.3.0.100.0-373.jar:/usr/hdp/3.0.100.0-373/tez/lib/jersey-client-1.19.jar:/usr/hdp/3.0.100.0-373/tez/lib/jersey-json-1.19.jar:/usr/hdp/3.0.100.0-373/tez/lib/jettison-1.3.4.jar:/usr/hdp/3.0.100.0-373/tez/lib/jetty-server-9.3.22.v20171030.jar:/usr/hdp/3.0.100.0-373/tez/lib/jetty-util-9.3.22.v20171030.jar:/usr/hdp/3.0.100.0-373/tez/lib/jsr305-3.0.0.jar:/usr/hdp/3.0.100.0-373/tez/lib/metrics-core-3.1.0.jar:/usr/hdp/3.0.100.0-373/tez/lib/protobuf-java-2.5.0.jar:/usr/hdp/3.0.100.0-373/tez/lib/servlet-api-2.5.jar:/usr/hdp/3.0.100.0-373/tez/lib/slf4j-api-1.7.10.jar:/usr/hdp/3.0.100.0-373/tez/*:/usr/hdp/3.0.100.0-373/tez/lib/*' ++ SCHEMATOOL_COMMAND='/usr/hdp/3.0.100.0-373/hive/bin/schematool -dbType postgres' ++ '[' hiveserver2 == metastore ']' ++ [[ hiveserver2 == \m\e\t\a\s\t\o\r\e ]] ++ [[ hiveserver2 == \h\i\v\e\s\e\r\v\e\r\2 ]] ++ export 'HADOOP_CLIENT_OPTS=-Djdk.tls.disabledAlgorithms=SSLv3,GCM -Xms8192M -Xmx13107M -Xloggc:/var/log/hive/hiveserver2-gc-%t.log -XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCCause -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M -javaagent:/jmx-exporter/jmx_prometheus_javaagent-0.11.0.jar=35000:/jmx-exporter/config.yml -agentlib:jdwp=transport=dt_socket,address=50505,suspend=n,server=y' ++ HADOOP_CLIENT_OPTS='-Djdk.tls.disabledAlgorithms=SSLv3,GCM -Xms8192M -Xmx13107M -Xloggc:/var/log/hive/hiveserver2-gc-%t.log -XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCCause -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M -javaagent:/jmx-exporter/jmx_prometheus_javaagent-0.11.0.jar=35000:/jmx-exporter/config.yml -agentlib:jdwp=transport=dt_socket,address=50505,suspend=n,server=y' ++ exec /usr/hdp/3.0.100.0-373/hive/bin/hive --skiphadoopversion --skiphbasecp --service hiveserver2 +2019-03-21 07:06:10: Starting HiveServer2 +Listening for transport dt_socket at address: 50505 +SLF4J: Class path contains multiple SLF4J bindings. +SLF4J: Found binding in [jar:file:/usr/hdp/3.0.100.0-373/hive/lib/log4j-slf4j-impl-2.10.0.jar!/org/slf4j/impl/StaticLoggerBinder.class] +SLF4J: Found binding in [jar:file:/usr/hdp/3.0.100.0-373/hadoop/lib/slf4j-log4j12-1.7.25.jar!/org/slf4j/impl/StaticLoggerBinder.class] +SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation. +SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory] +<14>1 2019-03-21T07:06:14.750Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="main"] Found configuration file file:/usr/hdp/3.0.100.0-373/hive/conf/hive-site.xml +<12>1 2019-03-21T07:06:15.136Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="WARN" thread="main"] HiveConf of name hive.stats.fetch.partition.stats does not exist +<12>1 2019-03-21T07:06:15.137Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="WARN" thread="main"] HiveConf of name hive.metastore.runworker.in does not exist +<12>1 2019-03-21T07:06:15.137Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="WARN" thread="main"] HiveConf of name hive.heapsize does not exist +<14>1 2019-03-21T07:06:15.248Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="server.HiveServer2" level="INFO" thread="main"] Starting HiveServer2 +<12>1 2019-03-21T07:06:15.284Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="WARN" thread="main"] HiveConf of name hive.stats.fetch.partition.stats does not exist +<12>1 2019-03-21T07:06:15.284Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="WARN" thread="main"] HiveConf of name hive.metastore.runworker.in does not exist +<12>1 2019-03-21T07:06:15.284Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="WARN" thread="main"] HiveConf of name hive.heapsize does not exist +Hive Session ID = b3203b63-408c-427b-bedf-68842b1aaa93 +<14>1 2019-03-21T07:06:15.432Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" thread="main"] Hive Session ID = b3203b63-408c-427b-bedf-68842b1aaa93 +<14>1 2019-03-21T07:06:16.027Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="impl.MetricsConfig" level="INFO" thread="main"] Loaded properties from hadoop-metrics2-s3a-file-system.properties +<14>1 2019-03-21T07:06:16.049Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="impl.MetricsSystemImpl" level="INFO" thread="main"] Scheduled Metric snapshot period at 10 second(s). +<14>1 2019-03-21T07:06:16.049Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="impl.MetricsSystemImpl" level="INFO" thread="main"] s3a-file-system metrics system started +<14>1 2019-03-21T07:06:17.680Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="main"] Created local directory: /tmp/hive +<14>1 2019-03-21T07:06:18.030Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="main"] Created HDFS directory: /tmp/hive/hive/b3203b63-408c-427b-bedf-68842b1aaa93 +<14>1 2019-03-21T07:06:18.034Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="main"] Created local directory: /tmp/hive/b3203b63-408c-427b-bedf-68842b1aaa93 +<14>1 2019-03-21T07:06:18.537Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="main"] Created HDFS directory: /tmp/hive/hive/b3203b63-408c-427b-bedf-68842b1aaa93/_tmp_space.db +<14>1 2019-03-21T07:06:18.552Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="sqlstd.SQLStdHiveAccessController" level="INFO" thread="main"] Created SQLStdHiveAccessController for session context : HiveAuthzSessionContext [sessionString=b3203b63-408c-427b-bedf-68842b1aaa93, clientType=HIVESERVER2] +<12>1 2019-03-21T07:06:18.555Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="WARN" thread="main"] METASTORE_FILTER_HOOK will be ignored, since hive.security.authorization.manager is set to instance of HiveAuthorizerFactory. +<14>1 2019-03-21T07:06:19.037Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="main"] Trying to connect to metastore with URI thrift://metastore-service.warehouse-1553110190-8zwt.svc.cluster.local:9083 +<14>1 2019-03-21T07:06:19.058Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="main"] Opened a connection to metastore, current connections: 1 +<14>1 2019-03-21T07:06:19.070Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="main"] Connected to metastore. +<14>1 2019-03-21T07:06:19.070Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.RetryingMetaStoreClient" level="INFO" thread="main"] RetryingMetaStoreClient proxy=class org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient ugi=hive (auth:SIMPLE) retries=24 delay=5 lifetime=0 +<14>1 2019-03-21T07:06:19.223Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="service.CompositeService" level="INFO" thread="main"] Operation log root directory is created: /tmp/hive/operation_logs +<14>1 2019-03-21T07:06:19.224Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="service.CompositeService" level="INFO" thread="main"] HiveServer2: Background operation thread pool size: 100 +<14>1 2019-03-21T07:06:19.224Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="service.CompositeService" level="INFO" thread="main"] HiveServer2: Background operation thread wait queue size: 100 +<14>1 2019-03-21T07:06:19.224Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="service.CompositeService" level="INFO" thread="main"] HiveServer2: Background operation thread keepalive time: 10 seconds +<14>1 2019-03-21T07:06:19.228Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="service.CompositeService" level="INFO" thread="main"] Connections limit are user: 0 ipaddress: 0 user-ipaddress: 0 +<14>1 2019-03-21T07:06:19.235Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="service.AbstractService" level="INFO" thread="main"] Service:OperationManager is inited. +<14>1 2019-03-21T07:06:19.236Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="service.AbstractService" level="INFO" thread="main"] Service:SessionManager is inited. +<14>1 2019-03-21T07:06:19.236Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="service.AbstractService" level="INFO" thread="main"] Service:CLIService is inited. +<14>1 2019-03-21T07:06:19.236Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="service.AbstractService" level="INFO" thread="main"] Service:ThriftHttpCLIService is inited. +<14>1 2019-03-21T07:06:19.236Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="service.AbstractService" level="INFO" thread="main"] Service:HiveServer2 is inited. +<14>1 2019-03-21T07:06:19.419Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="impl.ZkRegistryBase" level="INFO" thread="main"] Creating curator client with connectString: zookeeper.shared-services.svc.cluster.local:2181 sessionTimeoutMs: 120000 connectionTimeoutMs: 15000 namespace: llap-unsecure exponentialBackoff - sleepTime: 1000 maxRetries: 3 +<14>1 2019-03-21T07:06:19.498Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="impl.LlapZookeeperRegistryImpl" level="INFO" thread="main"] Llap Zookeeper Registry is enabled with registryid: compute-1553151951-nxph +<14>1 2019-03-21T07:06:19.498Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="impl.LlapRegistryService" level="INFO" thread="main"] Using LLAP registry type org.apache.hadoop.hive.llap.registry.impl.LlapZookeeperRegistryImpl@279e8bc0 +<14>1 2019-03-21T07:06:19.500Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="impl.ZookeeperUtils" level="INFO" thread="main"] UGI security is not enabled, or no SASL context name. Skipping setting up ZK auth. +<14>1 2019-03-21T07:06:19.500Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="imps.CuratorFrameworkImpl" level="INFO" thread="main"] Starting +<14>1 2019-03-21T07:06:19.509Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZooKeeper" level="INFO" thread="main"] Client environment:zookeeper.version=3.4.6-373--1, built on 03/16/2019 02:48 GMT +<14>1 2019-03-21T07:06:19.509Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZooKeeper" level="INFO" thread="main"] Client environment:host.name=hiveserver2-99ddccbd8-49sbv +<14>1 2019-03-21T07:06:19.509Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZooKeeper" level="INFO" thread="main"] Client environment:java.version=1.8.0_181 +<14>1 2019-03-21T07:06:19.509Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZooKeeper" level="INFO" thread="main"] Client environment:java.vendor=Oracle Corporation +<14>1 2019-03-21T07:06:19.509Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZooKeeper" level="INFO" thread="main"] Client environment:java.home=/usr/lib/jvm/java-8-openjdk-amd64/jre +<14>1 2019-03-21T07:06:19.509Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZooKeeper" level="INFO" thread="main"] Client environment:java.library.path=:/usr/hdp/3.0.100.0-373/hadoop/lib/native/Linux-amd64-64:/usr/hdp/3.0.100.0-373/hadoop/lib/native +<14>1 2019-03-21T07:06:19.509Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZooKeeper" level="INFO" thread="main"] Client environment:java.io.tmpdir=/tmp +<14>1 2019-03-21T07:06:19.509Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZooKeeper" level="INFO" thread="main"] Client environment:java.compiler= +<14>1 2019-03-21T07:06:19.509Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZooKeeper" level="INFO" thread="main"] Client environment:os.name=Linux +<14>1 2019-03-21T07:06:19.509Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZooKeeper" level="INFO" thread="main"] Client environment:os.arch=amd64 +<14>1 2019-03-21T07:06:19.509Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZooKeeper" level="INFO" thread="main"] Client environment:os.version=4.14.88-88.76.amzn2.x86_64 +<14>1 2019-03-21T07:06:19.509Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZooKeeper" level="INFO" thread="main"] Client environment:user.name=hive +<14>1 2019-03-21T07:06:19.509Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZooKeeper" level="INFO" thread="main"] Client environment:user.home=/home/hive +<14>1 2019-03-21T07:06:19.509Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZooKeeper" level="INFO" thread="main"] Client environment:user.dir=/usr/hdp/3.0.100.0-373 +<14>1 2019-03-21T07:06:19.510Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZooKeeper" level="INFO" thread="main"] Initiating client connection, connectString=zookeeper.shared-services.svc.cluster.local:2181 sessionTimeout=120000 watcher=org.apache.curator.ConnectionState@104cf647 +<14>1 2019-03-21T07:06:19.538Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ClientCnxn" level="INFO" thread="main-SendThread(zookeeper.shared-services.svc.cluster.local:2181)"] Opening socket connection to server zookeeper.shared-services.svc.cluster.local/10.100.96.143:2181. Will not attempt to authenticate using SASL (unknown error) +<14>1 2019-03-21T07:06:19.538Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="impl.LlapRegistryService" level="INFO" thread="main"] Using LLAP registry (client) type: Service LlapRegistryService in state LlapRegistryService: STARTED +<14>1 2019-03-21T07:06:19.542Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ClientCnxn" level="INFO" thread="main-SendThread(zookeeper.shared-services.svc.cluster.local:2181)"] Socket connection established, initiating session, client: /192.168.67.179:58032, server: zookeeper.shared-services.svc.cluster.local/10.100.96.143:2181 +<14>1 2019-03-21T07:06:19.550Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ClientCnxn" level="INFO" thread="main-SendThread(zookeeper.shared-services.svc.cluster.local:2181)"] Session establishment complete on server zookeeper.shared-services.svc.cluster.local/10.100.96.143:2181, sessionid = 0x2697e4a75ce01d9, negotiated timeout = 40000 +<14>1 2019-03-21T07:06:19.557Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="state.ConnectionStateManager" level="INFO" thread="main-EventThread"] State change: CONNECTED +<14>1 2019-03-21T07:06:19.558Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="impl.ZkRegistryBase" level="INFO" thread="Curator-ConnectionStateManager-0"] Connection state change notification received. State: CONNECTED +<12>1 2019-03-21T07:06:19.798Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="WARN" thread="main"] HiveConf of name hive.stats.fetch.partition.stats does not exist +<12>1 2019-03-21T07:06:19.798Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="WARN" thread="main"] HiveConf of name hive.metastore.runworker.in does not exist +<12>1 2019-03-21T07:06:19.798Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="WARN" thread="main"] HiveConf of name hive.heapsize does not exist +<14>1 2019-03-21T07:06:19.799Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="main"] Mestastore configuration metastore.filter.hook changed from org.apache.hadoop.hive.ql.security.authorization.plugin.AuthorizationMetaStoreFilterHook to org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl +<14>1 2019-03-21T07:06:19.822Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="main"] Closed a connection to metastore, current connections: 0 +<14>1 2019-03-21T07:06:19.830Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="results.QueryResultsCache" level="INFO" thread="main"] Initializing query results cache at /tmp/hive/_resultscache_ +Hive Session ID = 32cc5630-d166-4791-b2cd-8719dc20fd98 +<14>1 2019-03-21T07:06:19.838Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="SessionState" level="INFO" thread="HiveMaterializedViewsRegistry-0"] Hive Session ID = 32cc5630-d166-4791-b2cd-8719dc20fd98 +<14>1 2019-03-21T07:06:20.083Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="results.QueryResultsCache" level="INFO" thread="main"] Query results cache: cacheDirectory /tmp/hive/_resultscache_/results-81c4875c-9326-4e1b-9199-2f1f0e7fa3ff, maxCacheSize 2147483648, maxEntrySize 10485760, maxEntryLifetime 3600000 +<14>1 2019-03-21T07:06:20.094Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="main"] Trying to connect to metastore with URI thrift://metastore-service.warehouse-1553110190-8zwt.svc.cluster.local:9083 +<14>1 2019-03-21T07:06:20.095Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="main"] Opened a connection to metastore, current connections: 1 +<14>1 2019-03-21T07:06:20.096Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="main"] Connected to metastore. +<14>1 2019-03-21T07:06:20.096Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.RetryingMetaStoreClient" level="INFO" thread="main"] RetryingMetaStoreClient proxy=class org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient ugi=hive (auth:SIMPLE) retries=24 delay=5 lifetime=0 +<14>1 2019-03-21T07:06:20.107Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="events.NotificationEventPoll" level="INFO" thread="main"] Initializing lastCheckedEventId to 1531 +<14>1 2019-03-21T07:06:20.112Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="compactor.CompactorThread" level="INFO" thread="main"] Starting compactor thread of type org.apache.hadoop.hive.ql.txn.compactor.Worker +<12>1 2019-03-21T07:06:20.142Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="WARN" thread="main"] HiveConf of name hive.internal.ss.authz.settings.applied.marker does not exist +<12>1 2019-03-21T07:06:20.143Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="WARN" thread="main"] HiveConf of name hive.stats.fetch.partition.stats does not exist +<12>1 2019-03-21T07:06:20.143Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="WARN" thread="main"] HiveConf of name hive.metastore.runworker.in does not exist +<12>1 2019-03-21T07:06:20.143Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="WARN" thread="main"] HiveConf of name hive.heapsize does not exist +<14>1 2019-03-21T07:06:20.149Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="main"] Trying to connect to metastore with URI thrift://metastore-service.warehouse-1553110190-8zwt.svc.cluster.local:9083 +<14>1 2019-03-21T07:06:20.150Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="main"] Opened a connection to metastore, current connections: 2 +<14>1 2019-03-21T07:06:20.151Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="main"] Connected to metastore. +<14>1 2019-03-21T07:06:20.151Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.RetryingMetaStoreClient" level="INFO" thread="main"] RetryingMetaStoreClient proxy=class org.apache.hadoop.hive.metastore.HiveMetaStoreClient ugi=hive (auth:SIMPLE) retries=24 delay=5 lifetime=0 +<14>1 2019-03-21T07:06:20.170Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="server.HiveServer2" level="INFO" thread="main"] Starting Web UI on port 10100 +<14>1 2019-03-21T07:06:20.170Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="compactor.Worker" level="INFO" thread="hiveserver2-99ddccbd8-49sbv-31"] Starting Worker thread +<14>1 2019-03-21T07:06:20.179Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="server.HiveServer2" level="INFO" thread="main"] CORS enabled - allowed-origins: * allowed-methods: GET,POST,DELETE,HEAD allowed-headers: X-Requested-With,Content-Type,Accept,Origin,X-Requested-By,x-requested-by +<14>1 2019-03-21T07:06:20.345Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="service.AbstractService" level="INFO" thread="main"] Service:OperationManager is started. +<14>1 2019-03-21T07:06:20.345Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="service.AbstractService" level="INFO" thread="main"] Service:SessionManager is started. +<14>1 2019-03-21T07:06:20.347Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="service.AbstractService" level="INFO" thread="main"] Service:CLIService is started. +<14>1 2019-03-21T07:06:20.347Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="service.AbstractService" level="INFO" thread="main"] Service:ThriftHttpCLIService is started. +<14>1 2019-03-21T07:06:20.400Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveMaterializedViewsRegistry-0"] Created HDFS directory: /tmp/hive/hive/32cc5630-d166-4791-b2cd-8719dc20fd98 +<14>1 2019-03-21T07:06:20.405Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveMaterializedViewsRegistry-0"] Created local directory: /tmp/hive/32cc5630-d166-4791-b2cd-8719dc20fd98 +<12>1 2019-03-21T07:06:20.405Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="WARN" thread="main"] HiveConf of name hive.stats.fetch.partition.stats does not exist +<12>1 2019-03-21T07:06:20.405Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="WARN" thread="main"] HiveConf of name hive.metastore.runworker.in does not exist +<12>1 2019-03-21T07:06:20.405Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="WARN" thread="main"] HiveConf of name hive.heapsize does not exist +<12>1 2019-03-21T07:06:20.408Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="thrift.ThriftCLIService" level="WARN" thread="main"] XSRF filter disabled +<14>1 2019-03-21T07:06:20.458Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="thrift.ThriftCLIService" level="INFO" thread="main"] Started ThriftHttpCLIService in http mode on port 10010 path=/cliservice/* with 5...500 worker threads +<14>1 2019-03-21T07:06:20.459Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="service.AbstractService" level="INFO" thread="main"] Service:HiveServer2 is started. +<14>1 2019-03-21T07:06:20.547Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="http.CrossOriginFilter" level="INFO" thread="main"] Allowed Methods: GET,POST,DELETE,HEAD +<14>1 2019-03-21T07:06:20.547Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="http.CrossOriginFilter" level="INFO" thread="main"] Allowed Headers: X-Requested-With,Content-Type,Accept,Origin,X-Requested-By,x-requested-by +<14>1 2019-03-21T07:06:20.548Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="http.CrossOriginFilter" level="INFO" thread="main"] Allowed Origins: * +<14>1 2019-03-21T07:06:20.548Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="http.CrossOriginFilter" level="INFO" thread="main"] Allow All Origins: true +<14>1 2019-03-21T07:06:20.549Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="http.CrossOriginFilter" level="INFO" thread="main"] Max Age: 1800 +<14>1 2019-03-21T07:06:20.572Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="server.HiveServer2" level="INFO" thread="main"] Web UI has started on port 10100 +<14>1 2019-03-21T07:06:20.572Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="http.HttpServer" level="INFO" thread="main"] Started HttpServer[hiveserver2] on port 10100 +<14>1 2019-03-21T07:06:20.572Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="server.HiveServer2" level="INFO" thread="main"] HS2 interactive HA not enabled. Starting tez sessions.. +<14>1 2019-03-21T07:06:20.572Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="server.HiveServer2" level="INFO" thread="main"] Starting/Reconnecting tez sessions.. +<14>1 2019-03-21T07:06:20.572Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="server.HiveServer2" level="INFO" thread="main"] Initializing tez session pool manager +<14>1 2019-03-21T07:06:20.587Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZkAMRegistryClient" level="INFO" thread="main"] Returning tez AM registry (1017502292) for namespace '/compute-1553151951-nxph' +<14>1 2019-03-21T07:06:20.599Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="imps.CuratorFrameworkImpl" level="INFO" thread="main"] Starting +<14>1 2019-03-21T07:06:20.599Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZooKeeper" level="INFO" thread="main"] Initiating client connection, connectString=zookeeper.shared-services.svc.cluster.local:2181 sessionTimeout=60000 watcher=org.apache.curator.ConnectionState@4dc5b887 +<14>1 2019-03-21T07:06:20.601Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ClientCnxn" level="INFO" thread="main-SendThread(zookeeper.shared-services.svc.cluster.local:2181)"] Opening socket connection to server zookeeper.shared-services.svc.cluster.local/10.100.96.143:2181. Will not attempt to authenticate using SASL (unknown error) +<14>1 2019-03-21T07:06:20.602Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ClientCnxn" level="INFO" thread="main-SendThread(zookeeper.shared-services.svc.cluster.local:2181)"] Socket connection established, initiating session, client: /192.168.67.179:58050, server: zookeeper.shared-services.svc.cluster.local/10.100.96.143:2181 +<14>1 2019-03-21T07:06:20.604Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ClientCnxn" level="INFO" thread="main-SendThread(zookeeper.shared-services.svc.cluster.local:2181)"] Session establishment complete on server zookeeper.shared-services.svc.cluster.local/10.100.96.143:2181, sessionid = 0x3697e4a75d101c1, negotiated timeout = 40000 +<14>1 2019-03-21T07:06:20.604Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="state.ConnectionStateManager" level="INFO" thread="main-EventThread"] State change: CONNECTED +<12>1 2019-03-21T07:06:20.626Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="utils.ZKPaths" level="WARN" thread="main"] The version of ZooKeeper being used doesn't support Container nodes. CreateMode.PERSISTENT will be used instead. +<14>1 2019-03-21T07:06:20.637Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezExternalSessionsRegistryClient" level="INFO" thread="main"] Found 0 initial external sessions. +<14>1 2019-03-21T07:06:20.637Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezExternalSessionsRegistryClient" level="INFO" thread="main"] Returning tez external AM registry (1613856232) for namespace '/compute-1553151951-nxph' +<14>1 2019-03-21T07:06:20.644Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezSessionPool" level="INFO" thread="main"] Creating 2 new sessions +<14>1 2019-03-21T07:06:20.669Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezSessionPoolManager" level="INFO" thread="main"] Created new tez session for queue: default with session id: 8c762e43-f77e-4102-9855-4a9020e0e802 +<14>1 2019-03-21T07:06:20.670Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezSessionState" level="INFO" thread="main"] User of session id 8c762e43-f77e-4102-9855-4a9020e0e802 is hive +<14>1 2019-03-21T07:06:20.680Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.DagUtils" level="INFO" thread="main"] Jar dir is null / directory doesn't exist. Choosing HIVE_INSTALL_DIR - /user/hive/.hiveJars +<14>1 2019-03-21T07:06:20.682Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveMaterializedViewsRegistry-0"] Created HDFS directory: /tmp/hive/hive/32cc5630-d166-4791-b2cd-8719dc20fd98/_tmp_space.db +<14>1 2019-03-21T07:06:20.723Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metadata.HiveMaterializedViewsRegistry" level="INFO" thread="HiveMaterializedViewsRegistry-0"] Materialized views registry has been initialized +<14>1 2019-03-21T07:06:20.982Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezSessionState" level="INFO" thread="main"] Computed sha: b834ce577df1db2653da8692682fad938fdc40b72e0c34eb1666ab592b8431b8 for file: file:/usr/hdp/3.0.100.0-373/hive/lib/hive-exec-3.1.0.3.0.100.0-373.jar of length: 40.12MB in 283 ms +<14>1 2019-03-21T07:06:21.020Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.DagUtils" level="INFO" thread="main"] Resource modification time: 1552898199198 for s3a://edws-llap-perf/user/hive/.hiveJars/hive-exec-3.1.0.3.0.100.0-373-b834ce577df1db2653da8692682fad938fdc40b72e0c34eb1666ab592b8431b8.jar +<14>1 2019-03-21T07:06:21.046Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.DagUtils" level="INFO" thread="main"] Jar dir is null / directory doesn't exist. Choosing HIVE_INSTALL_DIR - /user/hive/.hiveJars +<14>1 2019-03-21T07:06:21.052Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezSessionState" level="INFO" thread="main"] Computed sha: d8a21192bd4753f527d9a0cb79764c81c172264a1f3a09092a97ddd8f7671681 for file: file:/usr/hdp/3.0.100.0-373/hive/lib/hive-llap-tez-3.1.0.3.0.100.0-373.jar of length: 127.90KB in 1 ms +<14>1 2019-03-21T07:06:21.058Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.DagUtils" level="INFO" thread="main"] Resource modification time: 1552898199532 for s3a://edws-llap-perf/user/hive/.hiveJars/hive-llap-tez-3.1.0.3.0.100.0-373-d8a21192bd4753f527d9a0cb79764c81c172264a1f3a09092a97ddd8f7671681.jar +<14>1 2019-03-21T07:06:21.067Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.DagUtils" level="INFO" thread="main"] Jar dir is null / directory doesn't exist. Choosing HIVE_INSTALL_DIR - /user/hive/.hiveJars +<14>1 2019-03-21T07:06:21.076Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.DagUtils" level="INFO" thread="main"] Resource modification time: 1552898199198 for s3a://edws-llap-perf/user/hive/.hiveJars/hive-exec-3.1.0.3.0.100.0-373-b834ce577df1db2653da8692682fad938fdc40b72e0c34eb1666ab592b8431b8.jar +<14>1 2019-03-21T07:06:21.084Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.DagUtils" level="INFO" thread="main"] Jar dir is null / directory doesn't exist. Choosing HIVE_INSTALL_DIR - /user/hive/.hiveJars +<14>1 2019-03-21T07:06:21.093Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.DagUtils" level="INFO" thread="main"] Resource modification time: 1552898199198 for s3a://edws-llap-perf/user/hive/.hiveJars/hive-exec-3.1.0.3.0.100.0-373-b834ce577df1db2653da8692682fad938fdc40b72e0c34eb1666ab592b8431b8.jar +<14>1 2019-03-21T07:06:21.101Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.DagUtils" level="INFO" thread="main"] Jar dir is null / directory doesn't exist. Choosing HIVE_INSTALL_DIR - /user/hive/.hiveJars +<14>1 2019-03-21T07:06:21.106Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezSessionState" level="INFO" thread="main"] Computed sha: 1f46aa93088b282475dfcf9e9aac6d3117ebe2a31326d62e80c5ad64897ec6a1 for file: file:/usr/hdp/3.0.100.0-373/hadoop-yarn/hadoop-yarn-registry-3.1.1.3.0.100.0-373.jar of length: 220.42KB in 2 ms +<14>1 2019-03-21T07:06:21.112Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.DagUtils" level="INFO" thread="main"] Resource modification time: 1552898199913 for s3a://edws-llap-perf/user/hive/.hiveJars/hadoop-yarn-registry-3.1.1.3.0.100.0-373-1f46aa93088b282475dfcf9e9aac6d3117ebe2a31326d62e80c5ad64897ec6a1.jar +<14>1 2019-03-21T07:06:21.210Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="client.TezClient" level="INFO" thread="main"] Tez Client Version: [ component=tez-api, version=0.9.1.3.0.100.0-373, revision=30cb59ca14dd336f63c629ce0e46c5fc6cc9b6aa, SCM-URL=scm:git:https://git-wip-us.apache.org/repos/asf/tez.git, buildTime=2019-03-16T03:34:26Z ] +<14>1 2019-03-21T07:06:21.211Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezSessionState" level="INFO" thread="main"] Opening new Tez Session (id: 8c762e43-f77e-4102-9855-4a9020e0e802) +<14>1 2019-03-21T07:06:22.035Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZkAMRegistryClient" level="INFO" thread="Curator-PathChildrenCache-0"] AppId allocated: /compute-1553151951-nxph/application_1553151980628_0000 +<14>1 2019-03-21T07:06:22.717Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZkAMRegistryClient" level="INFO" thread="Curator-PathChildrenCache-0"] AppId allocated: /compute-1553151951-nxph/application_1553151980628_0001 +<14>1 2019-03-21T07:06:25.432Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZkAMRegistryClient" level="INFO" thread="Curator-PathChildrenCache-0"] AM updated data: ServiceRecord{description='null'; external endpoints: {}; internal endpoints: {}, attributes: {"host"="query-coordinator-0.query-coordinator-service.compute-1553151951-nxph.svc.cluster.local" "externalId"="cb760f1c-4ba7-11e9-83d0-02c17ec35ac0" "port"="22222" "appId"="application_1553151980628_0000" }}. Notifying 1 listeners. +<14>1 2019-03-21T07:06:25.432Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezExternalSessionsRegistryClient" level="INFO" thread="Curator-PathChildrenCache-0"] Adding external session with applicationId: application_1553151980628_0000 +<14>1 2019-03-21T07:06:25.432Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezExternalSessionsRegistryClient" level="INFO" thread="main"] External session taken: application_1553151980628_0000. available: 0 +<14>1 2019-03-21T07:06:25.443Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZkAMRegistryClient" level="INFO" thread="main"] Returning tez AM registry (1017502292) for namespace '/compute-1553151951-nxph' +<14>1 2019-03-21T07:06:25.444Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="imps.CuratorFrameworkImpl" level="INFO" thread="main"] Starting +<14>1 2019-03-21T07:06:25.444Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZooKeeper" level="INFO" thread="main"] Initiating client connection, connectString=zookeeper.shared-services.svc.cluster.local:2181 sessionTimeout=60000 watcher=org.apache.curator.ConnectionState@4f9e9c21 +<14>1 2019-03-21T07:06:25.445Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ClientCnxn" level="INFO" thread="main-SendThread(zookeeper.shared-services.svc.cluster.local:2181)"] Opening socket connection to server zookeeper.shared-services.svc.cluster.local/10.100.96.143:2181. Will not attempt to authenticate using SASL (unknown error) +<14>1 2019-03-21T07:06:25.446Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ClientCnxn" level="INFO" thread="main-SendThread(zookeeper.shared-services.svc.cluster.local:2181)"] Socket connection established, initiating session, client: /192.168.67.179:58096, server: zookeeper.shared-services.svc.cluster.local/10.100.96.143:2181 +<14>1 2019-03-21T07:06:25.448Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ClientCnxn" level="INFO" thread="main-SendThread(zookeeper.shared-services.svc.cluster.local:2181)"] Session establishment complete on server zookeeper.shared-services.svc.cluster.local/10.100.96.143:2181, sessionid = 0x1697e4a79a801c6, negotiated timeout = 40000 +<14>1 2019-03-21T07:06:25.449Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="state.ConnectionStateManager" level="INFO" thread="main-EventThread"] State change: CONNECTED +<14>1 2019-03-21T07:06:25.468Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="client.TezClient" level="INFO" thread="main"] Session mode. Reconnecting to session: application_1553151980628_0000 +<14>1 2019-03-21T07:06:25.473Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="client.TezClientUtils" level="INFO" thread="main"] Ignoring 'tez.lib.uris' since 'tez.ignore.lib.uris' is set to true +<14>1 2019-03-21T07:06:25.495Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="client.TezClientUtils" level="INFO" thread="main"] Permissions on staging directory s3a://edws-llap-perf/tmp/hive/staging are incorrect: rwxrwxrwx. Fixing permissions to correct value rwx------ +<14>1 2019-03-21T07:06:25.806Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="client.TezClient" level="INFO" thread="main"] Tez system stage directory s3a://edws-llap-perf/tmp/hive/staging/.tez/application_1553151980628_0000 doesn't exist and is created +<14>1 2019-03-21T07:06:25.834Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.Configuration" level="INFO" thread="main"] resource-types.xml not found +<14>1 2019-03-21T07:06:25.835Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="resource.ResourceUtils" level="INFO" thread="main"] Unable to find 'resource-types.xml'. +<14>1 2019-03-21T07:06:25.837Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="common.TezYARNUtils" level="INFO" thread="main"] Ignoring 'tez.lib.uris' since 'tez.ignore.lib.uris' is set to true +<14>1 2019-03-21T07:06:25.914Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="Configuration.deprecation" level="INFO" thread="main"] yarn.resourcemanager.system-metrics-publisher.enabled is deprecated. Instead, use yarn.system-metrics-publisher.enabled +<14>1 2019-03-21T07:06:26.268Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZkAMRegistryClient" level="INFO" thread="Curator-PathChildrenCache-1"] AM updated data: ServiceRecord{description='null'; external endpoints: {}; internal endpoints: {}, attributes: {"host"="query-coordinator-1.query-coordinator-service.compute-1553151951-nxph.svc.cluster.local" "externalId"="cb775d91-4ba7-11e9-83d0-02c17ec35ac0" "port"="22222" "appId"="application_1553151980628_0001" }}. Notifying 1 listeners. +<14>1 2019-03-21T07:06:26.269Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezExternalSessionsRegistryClient" level="INFO" thread="Curator-PathChildrenCache-1"] Adding external session with applicationId: application_1553151980628_0001 +<14>1 2019-03-21T07:06:26.269Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZkAMRegistryClient" level="INFO" thread="Curator-PathChildrenCache-0"] AM updated data: ServiceRecord{description='null'; external endpoints: {}; internal endpoints: {}, attributes: {"host"="query-coordinator-1.query-coordinator-service.compute-1553151951-nxph.svc.cluster.local" "externalId"="cb775d91-4ba7-11e9-83d0-02c17ec35ac0" "port"="22222" "appId"="application_1553151980628_0001" }}. Notifying 1 listeners. +<14>1 2019-03-21T07:06:26.721Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="client.TezClient" level="INFO" thread="main"] The url to track the Tez Session: +<14>1 2019-03-21T07:06:26.722Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezSessionState" level="INFO" thread="main"] Started an external session; client name HIVE-8c762e43-f77e-4102-9855-4a9020e0e802, app ID application_1553151980628_0000 +<14>1 2019-03-21T07:06:26.723Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezSessionPoolManager" level="INFO" thread="main"] Created new tez session for queue: default with session id: 25ac1661-9279-4ebd-90c5-5a0a9b740837 +<14>1 2019-03-21T07:06:26.723Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezSessionState" level="INFO" thread="main"] User of session id 25ac1661-9279-4ebd-90c5-5a0a9b740837 is hive +<14>1 2019-03-21T07:06:26.736Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.DagUtils" level="INFO" thread="main"] Jar dir is null / directory doesn't exist. Choosing HIVE_INSTALL_DIR - /user/hive/.hiveJars +<14>1 2019-03-21T07:06:26.747Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.DagUtils" level="INFO" thread="main"] Resource modification time: 1552898199198 for s3a://edws-llap-perf/user/hive/.hiveJars/hive-exec-3.1.0.3.0.100.0-373-b834ce577df1db2653da8692682fad938fdc40b72e0c34eb1666ab592b8431b8.jar +<14>1 2019-03-21T07:06:26.753Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.DagUtils" level="INFO" thread="main"] Jar dir is null / directory doesn't exist. Choosing HIVE_INSTALL_DIR - /user/hive/.hiveJars +<14>1 2019-03-21T07:06:26.762Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.DagUtils" level="INFO" thread="main"] Resource modification time: 1552898199532 for s3a://edws-llap-perf/user/hive/.hiveJars/hive-llap-tez-3.1.0.3.0.100.0-373-d8a21192bd4753f527d9a0cb79764c81c172264a1f3a09092a97ddd8f7671681.jar +<14>1 2019-03-21T07:06:26.769Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.DagUtils" level="INFO" thread="main"] Jar dir is null / directory doesn't exist. Choosing HIVE_INSTALL_DIR - /user/hive/.hiveJars +<14>1 2019-03-21T07:06:26.778Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.DagUtils" level="INFO" thread="main"] Resource modification time: 1552898199198 for s3a://edws-llap-perf/user/hive/.hiveJars/hive-exec-3.1.0.3.0.100.0-373-b834ce577df1db2653da8692682fad938fdc40b72e0c34eb1666ab592b8431b8.jar +<14>1 2019-03-21T07:06:26.785Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.DagUtils" level="INFO" thread="main"] Jar dir is null / directory doesn't exist. Choosing HIVE_INSTALL_DIR - /user/hive/.hiveJars +<14>1 2019-03-21T07:06:26.793Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.DagUtils" level="INFO" thread="main"] Resource modification time: 1552898199198 for s3a://edws-llap-perf/user/hive/.hiveJars/hive-exec-3.1.0.3.0.100.0-373-b834ce577df1db2653da8692682fad938fdc40b72e0c34eb1666ab592b8431b8.jar +<14>1 2019-03-21T07:06:26.800Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.DagUtils" level="INFO" thread="main"] Jar dir is null / directory doesn't exist. Choosing HIVE_INSTALL_DIR - /user/hive/.hiveJars +<14>1 2019-03-21T07:06:26.810Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.DagUtils" level="INFO" thread="main"] Resource modification time: 1552898199913 for s3a://edws-llap-perf/user/hive/.hiveJars/hadoop-yarn-registry-3.1.1.3.0.100.0-373-1f46aa93088b282475dfcf9e9aac6d3117ebe2a31326d62e80c5ad64897ec6a1.jar +<14>1 2019-03-21T07:06:26.835Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="client.TezClient" level="INFO" thread="main"] Tez Client Version: [ component=tez-api, version=0.9.1.3.0.100.0-373, revision=30cb59ca14dd336f63c629ce0e46c5fc6cc9b6aa, SCM-URL=scm:git:https://git-wip-us.apache.org/repos/asf/tez.git, buildTime=2019-03-16T03:34:26Z ] +<14>1 2019-03-21T07:06:26.835Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezSessionState" level="INFO" thread="main"] Opening new Tez Session (id: 25ac1661-9279-4ebd-90c5-5a0a9b740837) +<14>1 2019-03-21T07:06:26.835Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezExternalSessionsRegistryClient" level="INFO" thread="main"] External session taken: application_1553151980628_0001. available: 0 +<14>1 2019-03-21T07:06:26.836Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZkAMRegistryClient" level="INFO" thread="main"] Returning tez AM registry (1017502292) for namespace '/compute-1553151951-nxph' +<14>1 2019-03-21T07:06:26.836Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="imps.CuratorFrameworkImpl" level="INFO" thread="main"] Starting +<14>1 2019-03-21T07:06:26.837Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ZooKeeper" level="INFO" thread="main"] Initiating client connection, connectString=zookeeper.shared-services.svc.cluster.local:2181 sessionTimeout=60000 watcher=org.apache.curator.ConnectionState@51d41b97 +<14>1 2019-03-21T07:06:26.838Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ClientCnxn" level="INFO" thread="main-SendThread(zookeeper.shared-services.svc.cluster.local:2181)"] Opening socket connection to server zookeeper.shared-services.svc.cluster.local/10.100.96.143:2181. Will not attempt to authenticate using SASL (unknown error) +<14>1 2019-03-21T07:06:26.839Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ClientCnxn" level="INFO" thread="main-SendThread(zookeeper.shared-services.svc.cluster.local:2181)"] Socket connection established, initiating session, client: /192.168.67.179:58106, server: zookeeper.shared-services.svc.cluster.local/10.100.96.143:2181 +<14>1 2019-03-21T07:06:26.841Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zookeeper.ClientCnxn" level="INFO" thread="main-SendThread(zookeeper.shared-services.svc.cluster.local:2181)"] Session establishment complete on server zookeeper.shared-services.svc.cluster.local/10.100.96.143:2181, sessionid = 0x2697e4a75ce01dd, negotiated timeout = 40000 +<14>1 2019-03-21T07:06:26.842Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="state.ConnectionStateManager" level="INFO" thread="main-EventThread"] State change: CONNECTED +<14>1 2019-03-21T07:06:26.852Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="client.TezClient" level="INFO" thread="main"] Session mode. Reconnecting to session: application_1553151980628_0001 +<14>1 2019-03-21T07:06:26.852Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="client.TezClientUtils" level="INFO" thread="main"] Ignoring 'tez.lib.uris' since 'tez.ignore.lib.uris' is set to true +<14>1 2019-03-21T07:06:26.865Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="client.TezClientUtils" level="INFO" thread="main"] Permissions on staging directory s3a://edws-llap-perf/tmp/hive/staging are incorrect: rwxrwxrwx. Fixing permissions to correct value rwx------ +<14>1 2019-03-21T07:06:27.085Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="client.TezClient" level="INFO" thread="main"] Tez system stage directory s3a://edws-llap-perf/tmp/hive/staging/.tez/application_1553151980628_0001 doesn't exist and is created +<14>1 2019-03-21T07:06:27.085Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="common.TezYARNUtils" level="INFO" thread="main"] Ignoring 'tez.lib.uris' since 'tez.ignore.lib.uris' is set to true +<14>1 2019-03-21T07:06:27.198Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="Configuration.deprecation" level="INFO" thread="main"] yarn.resourcemanager.system-metrics-publisher.enabled is deprecated. Instead, use yarn.system-metrics-publisher.enabled +<14>1 2019-03-21T07:06:27.632Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="client.TezClient" level="INFO" thread="main"] The url to track the Tez Session: +<14>1 2019-03-21T07:06:27.632Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezSessionState" level="INFO" thread="main"] Started an external session; client name HIVE-25ac1661-9279-4ebd-90c5-5a0a9b740837, app ID application_1553151980628_0001 +<14>1 2019-03-21T07:06:27.638Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="server.HiveServer2" level="INFO" thread="main"] Tez session pool manager initialized. +<14>1 2019-03-21T07:06:27.638Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="tez.TezSessionPoolSession" level="INFO" thread="main"] Started trigger validator with interval: 500 ms +<14>1 2019-03-21T07:06:27.638Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="server.HiveServer2" level="INFO" thread="main"] Workload management is not enabled. +<14>1 2019-03-21T07:06:39.980Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="thrift.ThriftHttpServlet" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Could not validate cookie sent, will try to generate a new cookie +<14>1 2019-03-21T07:06:40.134Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="thrift.ThriftHttpServlet" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Cookie added for clientUserName anonymous +<14>1 2019-03-21T07:06:40.145Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="thrift.ThriftCLIService" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Client protocol version: HIVE_CLI_SERVICE_PROTOCOL_V10 +<14>1 2019-03-21T07:06:40.484Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Created HDFS directory: /tmp/hive/hive/6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:40.488Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Created local directory: /tmp/hive/6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:40.701Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Created HDFS directory: /tmp/hive/hive/6b9cd2bb-5d9d-40a4-9276-171f53485440/_tmp_space.db +<14>1 2019-03-21T07:06:40.716Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Trying to connect to metastore with URI thrift://metastore-service.warehouse-1553110190-8zwt.svc.cluster.local:9083 +<14>1 2019-03-21T07:06:40.717Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Opened a connection to metastore, current connections: 3 +<14>1 2019-03-21T07:06:40.718Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Connected to metastore. +<14>1 2019-03-21T07:06:40.718Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.RetryingMetaStoreClient" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] RetryingMetaStoreClient proxy=class org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient ugi=hive (auth:SIMPLE) retries=24 delay=5 lifetime=0 +<14>1 2019-03-21T07:06:40.730Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.HiveSessionImpl" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Operation log session directory is created: /tmp/hive/operation_logs/6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:40.731Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="service.CompositeService" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Session opened, SessionHandle [6b9cd2bb-5d9d-40a4-9276-171f53485440], current sessions:1 +<14>1 2019-03-21T07:06:40.828Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:40.828Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Updating thread name to 6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:40.829Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:40.829Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-36"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:40.889Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:40.890Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Updating thread name to 6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35 +<14>1 2019-03-21T07:06:40.890Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:40.890Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-35 +<14>1 2019-03-21T07:06:40.897Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:40.897Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Updating thread name to 6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:40.914Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:40.915Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-36"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:40.946Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:40.946Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Updating thread name to 6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35 +<14>1 2019-03-21T07:06:40.972Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="operation.OperationManager" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35"] Adding operation: OperationHandle [opType=EXECUTE_STATEMENT, getHandleIdentifier()=c68fecdc-04ef-42b9-b90b-bcc2d4ab26c4] +<14>1 2019-03-21T07:06:40.981Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:40.981Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-35 +<15>1 2019-03-21T07:06:41.011Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] +<15>1 2019-03-21T07:06:41.013Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] +<15>1 2019-03-21T07:06:41.013Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] +<15>1 2019-03-21T07:06:41.013Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] +<14>1 2019-03-21T07:06:41.043Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] Compiling command(queryId=hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a): show databases +<15>1 2019-03-21T07:06:41.086Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] +<15>1 2019-03-21T07:06:41.688Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] +<15>1 2019-03-21T07:06:41.688Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] +<14>1 2019-03-21T07:06:41.729Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] Semantic Analysis Completed (retrial = false) +<15>1 2019-03-21T07:06:41.729Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] +<14>1 2019-03-21T07:06:41.777Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:database_name, type:string, comment:from deserializer)], properties:null) +<14>1 2019-03-21T07:06:41.837Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="exec.ListSinkOperator" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] Initializing operator LIST_SINK[0] +<15>1 2019-03-21T07:06:41.844Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] +<14>1 2019-03-21T07:06:41.846Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="sqlstd.SQLStdHiveAccessController" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] Created SQLStdHiveAccessController for session context : HiveAuthzSessionContext [sessionString=6b9cd2bb-5d9d-40a4-9276-171f53485440, clientType=HIVESERVER2] +<15>1 2019-03-21T07:06:41.849Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] +<15>1 2019-03-21T07:06:41.849Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] +<14>1 2019-03-21T07:06:41.850Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] Completed compiling command(queryId=hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a); Time taken: 0.836 seconds +<14>1 2019-03-21T07:06:41.853Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="reexec.ReExecDriver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] Execution #1 of query +<15>1 2019-03-21T07:06:41.854Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] +<15>1 2019-03-21T07:06:41.854Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] +<14>1 2019-03-21T07:06:41.854Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] Executing command(queryId=hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a): show databases +<15>1 2019-03-21T07:06:41.855Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] +<14>1 2019-03-21T07:06:41.859Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="hooks.HiveProtoLoggingHook" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] Event per file enabled: true +<15>1 2019-03-21T07:06:41.892Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] +<15>1 2019-03-21T07:06:41.892Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] +<15>1 2019-03-21T07:06:41.892Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] +<15>1 2019-03-21T07:06:41.897Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] +<14>1 2019-03-21T07:06:41.898Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] Starting task [Stage-0:DDL] in serial mode +<14>1 2019-03-21T07:06:41.916Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="exec.DDLTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] Found 4 database(s) matching the SHOW DATABASES statement. +<15>1 2019-03-21T07:06:41.924Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] +<15>1 2019-03-21T07:06:41.924Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] +<15>1 2019-03-21T07:06:41.924Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] +<15>1 2019-03-21T07:06:41.924Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] +<15>1 2019-03-21T07:06:41.925Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] +<15>1 2019-03-21T07:06:41.925Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] +OK +<14>1 2019-03-21T07:06:41.925Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] Completed executing command(queryId=hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a); Time taken: 0.071 seconds +<14>1 2019-03-21T07:06:41.925Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] OK +<15>1 2019-03-21T07:06:41.925Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a" sessionId="6b9cd2bb-5d9d-40a4-9276-171f53485440" thread="HiveServer2-Background-Pool: Thread-78"] +<14>1 2019-03-21T07:06:41.958Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:41.958Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Updating thread name to 6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35 +<14>1 2019-03-21T07:06:41.967Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:41.967Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-35 +<14>1 2019-03-21T07:06:42.037Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:42.037Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Updating thread name to 6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35 +<14>1 2019-03-21T07:06:42.037Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:42.037Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-35 +<14>1 2019-03-21T07:06:42.070Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:42.070Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Updating thread name to 6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:42.071Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:42.071Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-36"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:42.075Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:42.076Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Updating thread name to 6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35 +<14>1 2019-03-21T07:06:42.086Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="Configuration.deprecation" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35"] mapred.input.dir is deprecated. Instead, use mapreduce.input.fileinputformat.inputdir +<14>1 2019-03-21T07:06:42.195Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="mapred.FileInputFormat" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35"] Total input files to process : 1 +<14>1 2019-03-21T07:06:42.228Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="exec.ListSinkOperator" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35"] RECORDS_OUT_INTERMEDIATE:0, RECORDS_OUT_OPERATOR_LIST_SINK_0:4, +<14>1 2019-03-21T07:06:42.228Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:42.228Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-35 +<14>1 2019-03-21T07:06:42.231Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:42.231Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Updating thread name to 6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:42.232Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:42.232Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-36"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:42.234Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:42.234Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Updating thread name to 6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35 +<14>1 2019-03-21T07:06:42.235Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:42.235Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-35 +<14>1 2019-03-21T07:06:42.238Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:42.238Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Updating thread name to 6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:42.238Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:42.238Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-36"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:42.250Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:42.250Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Updating thread name to 6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35 +<14>1 2019-03-21T07:06:42.251Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="operation.OperationManager" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35"] Closing operation: OperationHandle [opType=EXECUTE_STATEMENT, getHandleIdentifier()=c68fecdc-04ef-42b9-b90b-bcc2d4ab26c4] +<14>1 2019-03-21T07:06:42.251Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="operation.OperationManager" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35"] Removed queryId: hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a corresponding to operation: OperationHandle [opType=EXECUTE_STATEMENT, getHandleIdentifier()=c68fecdc-04ef-42b9-b90b-bcc2d4ab26c4] with tag: null +<14>1 2019-03-21T07:06:42.269Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="operation.Operation" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35"] Closing operation log /tmp/hive/operation_logs/6b9cd2bb-5d9d-40a4-9276-171f53485440/hive_20190321070640_787a477f-ed2d-4f5f-a980-b1ef8063457a without delay +<14>1 2019-03-21T07:06:42.270Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:42.270Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-35"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-35 +<14>1 2019-03-21T07:06:42.298Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:42.298Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="service.CompositeService" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Session closed, SessionHandle [6b9cd2bb-5d9d-40a4-9276-171f53485440], current sessions:0 +<14>1 2019-03-21T07:06:42.298Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Updating thread name to 6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:42.299Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:42.299Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.HiveSessionImpl" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-36"] Operation log session directory is deleted: /tmp/hive/operation_logs/6b9cd2bb-5d9d-40a4-9276-171f53485440 +<14>1 2019-03-21T07:06:42.299Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="6b9cd2bb-5d9d-40a4-9276-171f53485440 HiveServer2-HttpHandler-Pool: Thread-36"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:42.522Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Deleted directory: /tmp/hive/hive/6b9cd2bb-5d9d-40a4-9276-171f53485440 on fs with scheme s3a +<14>1 2019-03-21T07:06:42.522Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Deleted directory: /tmp/hive/6b9cd2bb-5d9d-40a4-9276-171f53485440 on fs with scheme file +<14>1 2019-03-21T07:06:45.041Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="thrift.ThriftHttpServlet" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Could not validate cookie sent, will try to generate a new cookie +<14>1 2019-03-21T07:06:45.042Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="thrift.ThriftHttpServlet" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Cookie added for clientUserName anonymous +<14>1 2019-03-21T07:06:45.042Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="thrift.ThriftCLIService" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Client protocol version: HIVE_CLI_SERVICE_PROTOCOL_V10 +<14>1 2019-03-21T07:06:45.512Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Created HDFS directory: /tmp/hive/hive/5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:45.516Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Created local directory: /tmp/hive/5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:45.768Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Created HDFS directory: /tmp/hive/hive/5488e028-2bcb-4c28-af55-f3d49056fcb6/_tmp_space.db +<14>1 2019-03-21T07:06:45.769Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Trying to connect to metastore with URI thrift://metastore-service.warehouse-1553110190-8zwt.svc.cluster.local:9083 +<14>1 2019-03-21T07:06:45.770Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Opened a connection to metastore, current connections: 4 +<14>1 2019-03-21T07:06:45.771Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Connected to metastore. +<14>1 2019-03-21T07:06:45.771Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.RetryingMetaStoreClient" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] RetryingMetaStoreClient proxy=class org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient ugi=hive (auth:SIMPLE) retries=24 delay=5 lifetime=0 +<14>1 2019-03-21T07:06:45.798Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.HiveSessionImpl" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Operation log session directory is created: /tmp/hive/operation_logs/5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:45.798Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="service.CompositeService" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Session opened, SessionHandle [5488e028-2bcb-4c28-af55-f3d49056fcb6], current sessions:1 +<14>1 2019-03-21T07:06:45.874Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-37"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:45.874Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-37"] Updating thread name to 5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-37 +<14>1 2019-03-21T07:06:45.875Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-37"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:45.875Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-37"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-37 +<14>1 2019-03-21T07:06:45.884Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:45.884Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Updating thread name to 5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:45.885Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:45.885Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-36"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:45.892Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-37"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:45.892Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-37"] Updating thread name to 5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-37 +<14>1 2019-03-21T07:06:45.892Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-37"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:45.892Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-37"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-37 +<14>1 2019-03-21T07:06:45.928Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:45.928Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Updating thread name to 5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:45.929Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="operation.OperationManager" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-36"] Adding operation: OperationHandle [opType=EXECUTE_STATEMENT, getHandleIdentifier()=b5bf40aa-6843-4b71-a7fd-4b48f986e689] +<14>1 2019-03-21T07:06:45.930Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:45.930Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-36"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-36 +<15>1 2019-03-21T07:06:45.931Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] +<15>1 2019-03-21T07:06:45.931Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] +<15>1 2019-03-21T07:06:45.932Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] +<15>1 2019-03-21T07:06:45.932Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] +<14>1 2019-03-21T07:06:45.932Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] Compiling command(queryId=hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae): show databases +<15>1 2019-03-21T07:06:45.955Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] +<15>1 2019-03-21T07:06:45.956Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] +<15>1 2019-03-21T07:06:45.956Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] +<14>1 2019-03-21T07:06:45.958Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] Semantic Analysis Completed (retrial = false) +<15>1 2019-03-21T07:06:45.958Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] +<14>1 2019-03-21T07:06:45.959Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:database_name, type:string, comment:from deserializer)], properties:null) +<14>1 2019-03-21T07:06:45.960Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="exec.ListSinkOperator" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] Initializing operator LIST_SINK[0] +<15>1 2019-03-21T07:06:45.960Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] +<14>1 2019-03-21T07:06:45.960Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="sqlstd.SQLStdHiveAccessController" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] Created SQLStdHiveAccessController for session context : HiveAuthzSessionContext [sessionString=5488e028-2bcb-4c28-af55-f3d49056fcb6, clientType=HIVESERVER2] +<15>1 2019-03-21T07:06:45.960Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] +<15>1 2019-03-21T07:06:45.960Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] +<14>1 2019-03-21T07:06:45.960Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] Completed compiling command(queryId=hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae); Time taken: 0.028 seconds +<14>1 2019-03-21T07:06:45.961Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="reexec.ReExecDriver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] Execution #1 of query +<15>1 2019-03-21T07:06:45.961Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] +<15>1 2019-03-21T07:06:45.961Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] +<14>1 2019-03-21T07:06:45.961Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] Executing command(queryId=hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae): show databases +<15>1 2019-03-21T07:06:45.962Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] +<15>1 2019-03-21T07:06:45.962Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] +<15>1 2019-03-21T07:06:45.962Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] +<15>1 2019-03-21T07:06:45.962Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] +<15>1 2019-03-21T07:06:45.962Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] +<14>1 2019-03-21T07:06:45.962Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] Starting task [Stage-0:DDL] in serial mode +<14>1 2019-03-21T07:06:45.972Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="exec.DDLTask" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] Found 4 database(s) matching the SHOW DATABASES statement. +<15>1 2019-03-21T07:06:45.972Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] +<15>1 2019-03-21T07:06:45.972Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] +<15>1 2019-03-21T07:06:45.972Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] +<15>1 2019-03-21T07:06:45.972Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] +<15>1 2019-03-21T07:06:45.972Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] +<15>1 2019-03-21T07:06:45.972Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] +OK<14>1 2019-03-21T07:06:45.973Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] Completed executing command(queryId=hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae); Time taken: 0.011 seconds + +<14>1 2019-03-21T07:06:45.973Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] OK +<15>1 2019-03-21T07:06:45.973Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae" sessionId="5488e028-2bcb-4c28-af55-f3d49056fcb6" thread="HiveServer2-Background-Pool: Thread-81"] +<14>1 2019-03-21T07:06:45.997Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:45.997Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:45.997Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Updating thread name to 5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:45.997Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-36"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:46.020Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-37"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:46.020Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-37"] Updating thread name to 5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-37 +<14>1 2019-03-21T07:06:46.020Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-37"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:46.021Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-37"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-37 +<14>1 2019-03-21T07:06:46.040Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:46.040Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Updating thread name to 5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:46.041Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:46.041Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-36"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:46.045Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-37"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:46.045Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-37"] Updating thread name to 5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-37 +<14>1 2019-03-21T07:06:46.047Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="mapred.FileInputFormat" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-37"] Total input files to process : 1 +<14>1 2019-03-21T07:06:46.050Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="exec.ListSinkOperator" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-37"] RECORDS_OUT_INTERMEDIATE:0, RECORDS_OUT_OPERATOR_LIST_SINK_0:4, +<14>1 2019-03-21T07:06:46.050Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-37"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:46.050Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-37"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-37 +<14>1 2019-03-21T07:06:46.052Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:46.052Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Updating thread name to 5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:46.053Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:46.053Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-36"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:46.055Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-37"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:46.055Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-37"] Updating thread name to 5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-37 +<14>1 2019-03-21T07:06:46.055Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-37"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:46.055Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-37"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-37 +<14>1 2019-03-21T07:06:46.058Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:46.058Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Updating thread name to 5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:46.058Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:46.058Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-36"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:46.062Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-37"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:46.062Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-37"] Updating thread name to 5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-37 +<14>1 2019-03-21T07:06:46.062Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="operation.OperationManager" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-37"] Closing operation: OperationHandle [opType=EXECUTE_STATEMENT, getHandleIdentifier()=b5bf40aa-6843-4b71-a7fd-4b48f986e689] +<14>1 2019-03-21T07:06:46.062Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="operation.OperationManager" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-37"] Removed queryId: hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae corresponding to operation: OperationHandle [opType=EXECUTE_STATEMENT, getHandleIdentifier()=b5bf40aa-6843-4b71-a7fd-4b48f986e689] with tag: null +<14>1 2019-03-21T07:06:46.075Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="operation.Operation" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-37"] Closing operation log /tmp/hive/operation_logs/5488e028-2bcb-4c28-af55-f3d49056fcb6/hive_20190321070645_7a1b25c5-f106-4c0e-a8c9-6e1519e0c8ae without delay +<14>1 2019-03-21T07:06:46.075Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-37"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:46.075Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-37"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-37 +<14>1 2019-03-21T07:06:46.094Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="service.CompositeService" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Session closed, SessionHandle [5488e028-2bcb-4c28-af55-f3d49056fcb6], current sessions:0 +<14>1 2019-03-21T07:06:46.094Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:46.094Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Updating thread name to 5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:46.095Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.HiveSessionImpl" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-36"] Operation log session directory is deleted: /tmp/hive/operation_logs/5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:46.095Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 5488e028-2bcb-4c28-af55-f3d49056fcb6 +<14>1 2019-03-21T07:06:46.095Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="5488e028-2bcb-4c28-af55-f3d49056fcb6 HiveServer2-HttpHandler-Pool: Thread-36"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:46.276Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Deleted directory: /tmp/hive/hive/5488e028-2bcb-4c28-af55-f3d49056fcb6 on fs with scheme s3a +<14>1 2019-03-21T07:06:46.276Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Deleted directory: /tmp/hive/5488e028-2bcb-4c28-af55-f3d49056fcb6 on fs with scheme file +<14>1 2019-03-21T07:06:46.276Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Closed a connection to metastore, current connections: 3 +<14>1 2019-03-21T07:06:58.204Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="thrift.ThriftHttpServlet" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Could not validate cookie sent, will try to generate a new cookie +<14>1 2019-03-21T07:06:58.204Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="thrift.ThriftHttpServlet" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Cookie added for clientUserName hive +<14>1 2019-03-21T07:06:58.205Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="thrift.ThriftCLIService" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Client protocol version: HIVE_CLI_SERVICE_PROTOCOL_V10 +<14>1 2019-03-21T07:06:58.500Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Created HDFS directory: /tmp/hive/hive/00837dbe-3453-4bca-b8f5-b183fa7baed3 +<14>1 2019-03-21T07:06:58.504Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Created local directory: /tmp/hive/00837dbe-3453-4bca-b8f5-b183fa7baed3 +<14>1 2019-03-21T07:06:58.725Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Created HDFS directory: /tmp/hive/hive/00837dbe-3453-4bca-b8f5-b183fa7baed3/_tmp_space.db +<14>1 2019-03-21T07:06:58.734Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Trying to connect to metastore with URI thrift://metastore-service.warehouse-1553110190-8zwt.svc.cluster.local:9083 +<14>1 2019-03-21T07:06:58.734Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Opened a connection to metastore, current connections: 4 +<14>1 2019-03-21T07:06:58.735Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Connected to metastore. +<14>1 2019-03-21T07:06:58.735Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.RetryingMetaStoreClient" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] RetryingMetaStoreClient proxy=class org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient ugi=hive (auth:SIMPLE) retries=24 delay=5 lifetime=0 +<14>1 2019-03-21T07:06:58.741Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.HiveSessionImpl" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Operation log session directory is created: /tmp/hive/operation_logs/00837dbe-3453-4bca-b8f5-b183fa7baed3 +<14>1 2019-03-21T07:06:58.741Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="service.CompositeService" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Session opened, SessionHandle [00837dbe-3453-4bca-b8f5-b183fa7baed3], current sessions:1 +<14>1 2019-03-21T07:06:58.909Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Using the default value passed in for log id: 00837dbe-3453-4bca-b8f5-b183fa7baed3 +<14>1 2019-03-21T07:06:58.909Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Updating thread name to 00837dbe-3453-4bca-b8f5-b183fa7baed3 HiveServer2-HttpHandler-Pool: Thread-72 +<14>1 2019-03-21T07:06:58.909Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="00837dbe-3453-4bca-b8f5-b183fa7baed3 HiveServer2-HttpHandler-Pool: Thread-72"] Using the default value passed in for log id: 00837dbe-3453-4bca-b8f5-b183fa7baed3 +<14>1 2019-03-21T07:06:58.909Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="00837dbe-3453-4bca-b8f5-b183fa7baed3 HiveServer2-HttpHandler-Pool: Thread-72"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-72 +<14>1 2019-03-21T07:06:58.916Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 00837dbe-3453-4bca-b8f5-b183fa7baed3 +<14>1 2019-03-21T07:06:58.916Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-36"] Updating thread name to 00837dbe-3453-4bca-b8f5-b183fa7baed3 HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:58.916Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="00837dbe-3453-4bca-b8f5-b183fa7baed3 HiveServer2-HttpHandler-Pool: Thread-36"] Using the default value passed in for log id: 00837dbe-3453-4bca-b8f5-b183fa7baed3 +<14>1 2019-03-21T07:06:58.916Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="00837dbe-3453-4bca-b8f5-b183fa7baed3 HiveServer2-HttpHandler-Pool: Thread-36"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-36 +<14>1 2019-03-21T07:06:58.971Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="service.CompositeService" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Session closed, SessionHandle [00837dbe-3453-4bca-b8f5-b183fa7baed3], current sessions:0 +<14>1 2019-03-21T07:06:58.971Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Using the default value passed in for log id: 00837dbe-3453-4bca-b8f5-b183fa7baed3 +<14>1 2019-03-21T07:06:58.971Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Updating thread name to 00837dbe-3453-4bca-b8f5-b183fa7baed3 HiveServer2-HttpHandler-Pool: Thread-72 +<14>1 2019-03-21T07:06:58.971Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.HiveSessionImpl" level="INFO" thread="00837dbe-3453-4bca-b8f5-b183fa7baed3 HiveServer2-HttpHandler-Pool: Thread-72"] Operation log session directory is deleted: /tmp/hive/operation_logs/00837dbe-3453-4bca-b8f5-b183fa7baed3 +<14>1 2019-03-21T07:06:58.971Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="00837dbe-3453-4bca-b8f5-b183fa7baed3 HiveServer2-HttpHandler-Pool: Thread-72"] Using the default value passed in for log id: 00837dbe-3453-4bca-b8f5-b183fa7baed3 +<14>1 2019-03-21T07:06:58.971Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="00837dbe-3453-4bca-b8f5-b183fa7baed3 HiveServer2-HttpHandler-Pool: Thread-72"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-72 +<14>1 2019-03-21T07:06:59.125Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Deleted directory: /tmp/hive/hive/00837dbe-3453-4bca-b8f5-b183fa7baed3 on fs with scheme s3a +<14>1 2019-03-21T07:06:59.125Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Deleted directory: /tmp/hive/00837dbe-3453-4bca-b8f5-b183fa7baed3 on fs with scheme file +<14>1 2019-03-21T07:07:09.209Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="thrift.ThriftHttpServlet" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Could not validate cookie sent, will try to generate a new cookie +<14>1 2019-03-21T07:07:09.209Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="thrift.ThriftHttpServlet" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Cookie added for clientUserName admin +<14>1 2019-03-21T07:07:09.210Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="thrift.ThriftCLIService" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Client protocol version: HIVE_CLI_SERVICE_PROTOCOL_V10 +<14>1 2019-03-21T07:07:09.435Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Created HDFS directory: /tmp/hive/hive/839a405d-d76f-499f-b85c-be8a8bfa5b0a +<14>1 2019-03-21T07:07:09.437Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Created local directory: /tmp/hive/839a405d-d76f-499f-b85c-be8a8bfa5b0a +<14>1 2019-03-21T07:07:09.638Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Created HDFS directory: /tmp/hive/hive/839a405d-d76f-499f-b85c-be8a8bfa5b0a/_tmp_space.db +<14>1 2019-03-21T07:07:09.639Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Trying to connect to metastore with URI thrift://metastore-service.warehouse-1553110190-8zwt.svc.cluster.local:9083 +<14>1 2019-03-21T07:07:09.640Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Opened a connection to metastore, current connections: 5 +<14>1 2019-03-21T07:07:09.641Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Connected to metastore. +<14>1 2019-03-21T07:07:09.641Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.RetryingMetaStoreClient" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] RetryingMetaStoreClient proxy=class org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient ugi=hive (auth:SIMPLE) retries=24 delay=5 lifetime=0 +<14>1 2019-03-21T07:07:09.647Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.HiveSessionImpl" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Operation log session directory is created: /tmp/hive/operation_logs/839a405d-d76f-499f-b85c-be8a8bfa5b0a +<14>1 2019-03-21T07:07:09.647Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="service.CompositeService" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Session opened, SessionHandle [839a405d-d76f-499f-b85c-be8a8bfa5b0a], current sessions:1 +<14>1 2019-03-21T07:07:09.665Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-34"] Using the default value passed in for log id: 839a405d-d76f-499f-b85c-be8a8bfa5b0a +<14>1 2019-03-21T07:07:09.665Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-34"] Updating thread name to 839a405d-d76f-499f-b85c-be8a8bfa5b0a HiveServer2-HttpHandler-Pool: Thread-34 +<14>1 2019-03-21T07:07:09.666Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="operation.OperationManager" level="INFO" thread="839a405d-d76f-499f-b85c-be8a8bfa5b0a HiveServer2-HttpHandler-Pool: Thread-34"] Adding operation: OperationHandle [opType=EXECUTE_STATEMENT, getHandleIdentifier()=80a34e8e-2778-4d56-967d-676c5933f5ff] +<14>1 2019-03-21T07:07:09.667Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="839a405d-d76f-499f-b85c-be8a8bfa5b0a HiveServer2-HttpHandler-Pool: Thread-34"] Using the default value passed in for log id: 839a405d-d76f-499f-b85c-be8a8bfa5b0a +<14>1 2019-03-21T07:07:09.667Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="839a405d-d76f-499f-b85c-be8a8bfa5b0a HiveServer2-HttpHandler-Pool: Thread-34"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-34 +<15>1 2019-03-21T07:07:09.669Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] +<15>1 2019-03-21T07:07:09.669Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] +<15>1 2019-03-21T07:07:09.669Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] +<15>1 2019-03-21T07:07:09.669Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] +<14>1 2019-03-21T07:07:09.669Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] Compiling command(queryId=hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4): use default +<15>1 2019-03-21T07:07:09.695Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] +<15>1 2019-03-21T07:07:09.696Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] +<15>1 2019-03-21T07:07:09.696Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] +<14>1 2019-03-21T07:07:09.697Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] Trying to connect to metastore with URI thrift://metastore-service.warehouse-1553110190-8zwt.svc.cluster.local:9083 +<14>1 2019-03-21T07:07:09.697Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] Opened a connection to metastore, current connections: 6 +<14>1 2019-03-21T07:07:09.698Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] Connected to metastore. +<14>1 2019-03-21T07:07:09.698Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.RetryingMetaStoreClient" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] RetryingMetaStoreClient proxy=class org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient ugi=hive (auth:SIMPLE) retries=24 delay=5 lifetime=0 +<14>1 2019-03-21T07:07:09.707Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] Semantic Analysis Completed (retrial = false) +<15>1 2019-03-21T07:07:09.707Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] +<14>1 2019-03-21T07:07:09.707Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] Returning Hive schema: Schema(fieldSchemas:null, properties:null) +<15>1 2019-03-21T07:07:09.707Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] +<14>1 2019-03-21T07:07:09.707Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="sqlstd.SQLStdHiveAccessController" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] Created SQLStdHiveAccessController for session context : HiveAuthzSessionContext [sessionString=839a405d-d76f-499f-b85c-be8a8bfa5b0a, clientType=HIVESERVER2] +<15>1 2019-03-21T07:07:09.709Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] +<15>1 2019-03-21T07:07:09.709Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] +<14>1 2019-03-21T07:07:09.709Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] Completed compiling command(queryId=hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4); Time taken: 0.04 seconds +<14>1 2019-03-21T07:07:09.711Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="reexec.ReExecDriver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] Execution #1 of query +<15>1 2019-03-21T07:07:09.711Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] +<15>1 2019-03-21T07:07:09.711Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] +<14>1 2019-03-21T07:07:09.711Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] Executing command(queryId=hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4): use default +<15>1 2019-03-21T07:07:09.711Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] +<15>1 2019-03-21T07:07:09.711Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] +<15>1 2019-03-21T07:07:09.711Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] +<15>1 2019-03-21T07:07:09.711Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] +<15>1 2019-03-21T07:07:09.711Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] +<14>1 2019-03-21T07:07:09.711Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] Starting task [Stage-0:DDL] in serial mode +<15>1 2019-03-21T07:07:09.715Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] +<15>1 2019-03-21T07:07:09.716Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] +<15>1 2019-03-21T07:07:09.716Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] +<15>1 2019-03-21T07:07:09.716Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] +<15>1 2019-03-21T07:07:09.716Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] +OK +<15>1 2019-03-21T07:07:09.716Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] +<14>1 2019-03-21T07:07:09.716Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] Completed executing command(queryId=hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4); Time taken: 0.005 seconds +<14>1 2019-03-21T07:07:09.716Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] OK +<15>1 2019-03-21T07:07:09.716Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9e1a2117-1068-46d3-9de7-328358c203d4" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-83"] +<14>1 2019-03-21T07:07:09.747Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Using the default value passed in for log id: 839a405d-d76f-499f-b85c-be8a8bfa5b0a +<14>1 2019-03-21T07:07:09.747Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Updating thread name to 839a405d-d76f-499f-b85c-be8a8bfa5b0a HiveServer2-HttpHandler-Pool: Thread-72 +<14>1 2019-03-21T07:07:09.747Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="839a405d-d76f-499f-b85c-be8a8bfa5b0a HiveServer2-HttpHandler-Pool: Thread-72"] Using the default value passed in for log id: 839a405d-d76f-499f-b85c-be8a8bfa5b0a +<14>1 2019-03-21T07:07:09.747Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="839a405d-d76f-499f-b85c-be8a8bfa5b0a HiveServer2-HttpHandler-Pool: Thread-72"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-72 +<14>1 2019-03-21T07:07:09.760Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-34"] Using the default value passed in for log id: 839a405d-d76f-499f-b85c-be8a8bfa5b0a +<14>1 2019-03-21T07:07:09.760Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-34"] Updating thread name to 839a405d-d76f-499f-b85c-be8a8bfa5b0a HiveServer2-HttpHandler-Pool: Thread-34 +<14>1 2019-03-21T07:07:09.761Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="operation.OperationManager" level="INFO" thread="839a405d-d76f-499f-b85c-be8a8bfa5b0a HiveServer2-HttpHandler-Pool: Thread-34"] Adding operation: OperationHandle [opType=EXECUTE_STATEMENT, getHandleIdentifier()=1fefbcb6-4ada-443e-80f2-cce1290b4c4c] +<14>1 2019-03-21T07:07:09.761Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="839a405d-d76f-499f-b85c-be8a8bfa5b0a HiveServer2-HttpHandler-Pool: Thread-34"] Using the default value passed in for log id: 839a405d-d76f-499f-b85c-be8a8bfa5b0a +<14>1 2019-03-21T07:07:09.762Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="839a405d-d76f-499f-b85c-be8a8bfa5b0a HiveServer2-HttpHandler-Pool: Thread-34"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-34 +<15>1 2019-03-21T07:07:09.762Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<15>1 2019-03-21T07:07:09.762Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<15>1 2019-03-21T07:07:09.762Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<15>1 2019-03-21T07:07:09.763Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<14>1 2019-03-21T07:07:09.763Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] Compiling command(queryId=hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682): drop table ttt +<15>1 2019-03-21T07:07:09.779Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<15>1 2019-03-21T07:07:09.779Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<15>1 2019-03-21T07:07:09.780Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<14>1 2019-03-21T07:07:09.798Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="lockmgr.DbTxnManager" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] Opened txnid:668 +<14>1 2019-03-21T07:07:09.823Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] Semantic Analysis Completed (retrial = false) +<15>1 2019-03-21T07:07:09.823Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<14>1 2019-03-21T07:07:09.823Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] Returning Hive schema: Schema(fieldSchemas:null, properties:null) +<15>1 2019-03-21T07:07:09.823Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<15>1 2019-03-21T07:07:09.823Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<15>1 2019-03-21T07:07:09.823Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<14>1 2019-03-21T07:07:09.823Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] Completed compiling command(queryId=hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682); Time taken: 0.06 seconds +<14>1 2019-03-21T07:07:09.825Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="reexec.ReExecDriver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] Execution #1 of query +<15>1 2019-03-21T07:07:09.825Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<14>1 2019-03-21T07:07:09.826Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="lockmgr.DbTxnManager" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] Setting lock request transaction to txnid:668 for queryId=hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682 +<15>1 2019-03-21T07:07:09.826Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<15>1 2019-03-21T07:07:09.828Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<14>1 2019-03-21T07:07:09.828Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] Executing command(queryId=hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682): drop table ttt +<15>1 2019-03-21T07:07:09.828Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<14>1 2019-03-21T07:07:09.829Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="hooks.HiveProtoLoggingHook" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] Received pre-hook notification for: hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682 +<14>1 2019-03-21T07:07:09.845Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] Using the default value passed in for log id: 839a405d-d76f-499f-b85c-be8a8bfa5b0a +<15>1 2019-03-21T07:07:09.854Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<15>1 2019-03-21T07:07:09.854Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<15>1 2019-03-21T07:07:09.854Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<15>1 2019-03-21T07:07:09.854Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<14>1 2019-03-21T07:07:09.854Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] Starting task [Stage-0:DDL] in serial mode +<11>1 2019-03-21T07:07:09.857Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metadata.Hive" level="ERROR" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] Table default.ttt not found: hive.default.ttt table not found +<15>1 2019-03-21T07:07:09.871Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<15>1 2019-03-21T07:07:09.871Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<15>1 2019-03-21T07:07:09.871Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<15>1 2019-03-21T07:07:09.871Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<14>1 2019-03-21T07:07:09.871Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="hooks.HiveProtoLoggingHook" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] Received post-hook notification for: hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682 +<15>1 2019-03-21T07:07:09.872Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<15>1 2019-03-21T07:07:09.872Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +OK +<14>1 2019-03-21T07:07:09.872Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] Completed executing command(queryId=hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682); Time taken: 0.044 seconds +<14>1 2019-03-21T07:07:09.872Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] OK +<15>1 2019-03-21T07:07:09.872Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<14>1 2019-03-21T07:07:09.873Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="lockmgr.DbTxnManager" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] Stopped heartbeat for query: hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682 +<15>1 2019-03-21T07:07:09.883Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<15>1 2019-03-21T07:07:09.883Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070709_9f2f03d6-fa39-48d9-b10d-732cbe62d682" sessionId="839a405d-d76f-499f-b85c-be8a8bfa5b0a" thread="HiveServer2-Background-Pool: Thread-85"] +<14>1 2019-03-21T07:07:09.990Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Using the default value passed in for log id: 839a405d-d76f-499f-b85c-be8a8bfa5b0a +<14>1 2019-03-21T07:07:09.990Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Updating thread name to 839a405d-d76f-499f-b85c-be8a8bfa5b0a HiveServer2-HttpHandler-Pool: Thread-72 +<14>1 2019-03-21T07:07:09.991Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="839a405d-d76f-499f-b85c-be8a8bfa5b0a HiveServer2-HttpHandler-Pool: Thread-72"] Using the default value passed in for log id: 839a405d-d76f-499f-b85c-be8a8bfa5b0a +<14>1 2019-03-21T07:07:09.991Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="839a405d-d76f-499f-b85c-be8a8bfa5b0a HiveServer2-HttpHandler-Pool: Thread-72"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-72 +<14>1 2019-03-21T07:07:10.026Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="zlib.ZlibFactory" level="INFO" thread="Hive Hook Proto Log Writer 0"] Successfully loaded & initialized native-zlib library +<14>1 2019-03-21T07:07:10.027Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="compress.CodecPool" level="INFO" thread="Hive Hook Proto Log Writer 0"] Got brand-new compressor [.deflate] +<14>1 2019-03-21T07:07:16.469Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="thrift.ThriftHttpServlet" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Could not validate cookie sent, will try to generate a new cookie +<14>1 2019-03-21T07:07:16.469Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="thrift.ThriftHttpServlet" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Cookie added for clientUserName admin +<14>1 2019-03-21T07:07:16.469Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="thrift.ThriftCLIService" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Client protocol version: HIVE_CLI_SERVICE_PROTOCOL_V10 +<14>1 2019-03-21T07:07:16.715Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Created HDFS directory: /tmp/hive/hive/1ba9df58-94d3-4235-8920-cc4a8c7afd84 +<14>1 2019-03-21T07:07:16.719Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Created local directory: /tmp/hive/1ba9df58-94d3-4235-8920-cc4a8c7afd84 +<14>1 2019-03-21T07:07:16.922Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Created HDFS directory: /tmp/hive/hive/1ba9df58-94d3-4235-8920-cc4a8c7afd84/_tmp_space.db +<14>1 2019-03-21T07:07:16.926Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.HiveSessionImpl" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Operation log session directory is created: /tmp/hive/operation_logs/1ba9df58-94d3-4235-8920-cc4a8c7afd84 +<14>1 2019-03-21T07:07:16.926Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="service.CompositeService" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Session opened, SessionHandle [1ba9df58-94d3-4235-8920-cc4a8c7afd84], current sessions:2 +<14>1 2019-03-21T07:07:16.960Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Using the default value passed in for log id: 1ba9df58-94d3-4235-8920-cc4a8c7afd84 +<14>1 2019-03-21T07:07:16.960Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Updating thread name to 1ba9df58-94d3-4235-8920-cc4a8c7afd84 HiveServer2-HttpHandler-Pool: Thread-35 +<14>1 2019-03-21T07:07:16.962Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="operation.OperationManager" level="INFO" thread="1ba9df58-94d3-4235-8920-cc4a8c7afd84 HiveServer2-HttpHandler-Pool: Thread-35"] Adding operation: OperationHandle [opType=EXECUTE_STATEMENT, getHandleIdentifier()=b02ee978-b7fb-4d49-b522-507b35cd1f51] +<14>1 2019-03-21T07:07:16.962Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="1ba9df58-94d3-4235-8920-cc4a8c7afd84 HiveServer2-HttpHandler-Pool: Thread-35"] Using the default value passed in for log id: 1ba9df58-94d3-4235-8920-cc4a8c7afd84 +<14>1 2019-03-21T07:07:16.962Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="1ba9df58-94d3-4235-8920-cc4a8c7afd84 HiveServer2-HttpHandler-Pool: Thread-35"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-35 +<15>1 2019-03-21T07:07:16.964Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] +<15>1 2019-03-21T07:07:16.964Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] +<15>1 2019-03-21T07:07:16.964Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] +<15>1 2019-03-21T07:07:16.964Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] +<14>1 2019-03-21T07:07:16.964Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] Compiling command(queryId=hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06): use default +<15>1 2019-03-21T07:07:16.987Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] +<15>1 2019-03-21T07:07:16.987Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] +<15>1 2019-03-21T07:07:16.987Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] +<14>1 2019-03-21T07:07:16.991Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] Semantic Analysis Completed (retrial = false) +<15>1 2019-03-21T07:07:16.991Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] +<14>1 2019-03-21T07:07:16.991Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] Returning Hive schema: Schema(fieldSchemas:null, properties:null) +<15>1 2019-03-21T07:07:16.991Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] +<14>1 2019-03-21T07:07:16.991Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="sqlstd.SQLStdHiveAccessController" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] Created SQLStdHiveAccessController for session context : HiveAuthzSessionContext [sessionString=1ba9df58-94d3-4235-8920-cc4a8c7afd84, clientType=HIVESERVER2] +<15>1 2019-03-21T07:07:16.991Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] +<15>1 2019-03-21T07:07:16.992Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] +<14>1 2019-03-21T07:07:16.992Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] Completed compiling command(queryId=hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06); Time taken: 0.028 seconds +<14>1 2019-03-21T07:07:16.993Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="reexec.ReExecDriver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] Execution #1 of query +<15>1 2019-03-21T07:07:16.993Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] +<15>1 2019-03-21T07:07:16.993Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] +<14>1 2019-03-21T07:07:16.993Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] Executing command(queryId=hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06): use default +<15>1 2019-03-21T07:07:16.993Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] +<15>1 2019-03-21T07:07:16.993Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] +<15>1 2019-03-21T07:07:16.993Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] +<15>1 2019-03-21T07:07:16.993Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] +<15>1 2019-03-21T07:07:16.993Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] +<14>1 2019-03-21T07:07:16.993Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] Starting task [Stage-0:DDL] in serial mode +<15>1 2019-03-21T07:07:16.997Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] +<15>1 2019-03-21T07:07:16.997Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] +<15>1 2019-03-21T07:07:16.997Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] +<15>1 2019-03-21T07:07:16.997Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] +<15>1 2019-03-21T07:07:16.997Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] +<15>1 2019-03-21T07:07:16.998Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] +OK<14>1 2019-03-21T07:07:16.998Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] Completed executing command(queryId=hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06); Time taken: 0.005 seconds + +<14>1 2019-03-21T07:07:16.998Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] OK +<15>1 2019-03-21T07:07:16.998Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070716_f56aecf5-529e-441c-8a89-dd11ba757b06" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-90"] +<14>1 2019-03-21T07:07:17.004Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Using the default value passed in for log id: 1ba9df58-94d3-4235-8920-cc4a8c7afd84 +<14>1 2019-03-21T07:07:17.004Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-72"] Updating thread name to 1ba9df58-94d3-4235-8920-cc4a8c7afd84 HiveServer2-HttpHandler-Pool: Thread-72 +<14>1 2019-03-21T07:07:17.005Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="1ba9df58-94d3-4235-8920-cc4a8c7afd84 HiveServer2-HttpHandler-Pool: Thread-72"] Using the default value passed in for log id: 1ba9df58-94d3-4235-8920-cc4a8c7afd84 +<14>1 2019-03-21T07:07:17.005Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="1ba9df58-94d3-4235-8920-cc4a8c7afd84 HiveServer2-HttpHandler-Pool: Thread-72"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-72 +<14>1 2019-03-21T07:07:17.006Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Using the default value passed in for log id: 1ba9df58-94d3-4235-8920-cc4a8c7afd84 +<14>1 2019-03-21T07:07:17.007Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Updating thread name to 1ba9df58-94d3-4235-8920-cc4a8c7afd84 HiveServer2-HttpHandler-Pool: Thread-35 +<14>1 2019-03-21T07:07:17.007Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="operation.OperationManager" level="INFO" thread="1ba9df58-94d3-4235-8920-cc4a8c7afd84 HiveServer2-HttpHandler-Pool: Thread-35"] Adding operation: OperationHandle [opType=EXECUTE_STATEMENT, getHandleIdentifier()=d5df061b-6db8-4ffe-992a-7c929898db8a] +<14>1 2019-03-21T07:07:17.008Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="1ba9df58-94d3-4235-8920-cc4a8c7afd84 HiveServer2-HttpHandler-Pool: Thread-35"] Using the default value passed in for log id: 1ba9df58-94d3-4235-8920-cc4a8c7afd84 +<14>1 2019-03-21T07:07:17.008Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="1ba9df58-94d3-4235-8920-cc4a8c7afd84 HiveServer2-HttpHandler-Pool: Thread-35"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-35 +<15>1 2019-03-21T07:07:17.009Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070717_45f3313d-83e8-4994-a998-a4e60b19b136" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-92"] +<15>1 2019-03-21T07:07:17.009Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070717_45f3313d-83e8-4994-a998-a4e60b19b136" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-92"] +<15>1 2019-03-21T07:07:17.009Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070717_45f3313d-83e8-4994-a998-a4e60b19b136" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-92"] +<15>1 2019-03-21T07:07:17.009Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070717_45f3313d-83e8-4994-a998-a4e60b19b136" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-92"] +<14>1 2019-03-21T07:07:17.009Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070717_45f3313d-83e8-4994-a998-a4e60b19b136" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-92"] Compiling command(queryId=hive_20190321070717_45f3313d-83e8-4994-a998-a4e60b19b136): drop database ttt +<15>1 2019-03-21T07:07:17.021Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070717_45f3313d-83e8-4994-a998-a4e60b19b136" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-92"] +<15>1 2019-03-21T07:07:17.022Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070717_45f3313d-83e8-4994-a998-a4e60b19b136" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-92"] +<15>1 2019-03-21T07:07:17.022Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070717_45f3313d-83e8-4994-a998-a4e60b19b136" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-92"] +<14>1 2019-03-21T07:07:17.030Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="lockmgr.DbTxnManager" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070717_45f3313d-83e8-4994-a998-a4e60b19b136" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-92"] Opened txnid:669 +FAILED: SemanticException [Error 10072]: Database does not exist: ttt +<11>1 2019-03-21T07:07:17.035Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="ERROR" operationLogLevel="EXECUTION" queryId="hive_20190321070717_45f3313d-83e8-4994-a998-a4e60b19b136" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-92"] FAILED: SemanticException [Error 10072]: Database does not exist: ttt org.apache.hadoop.hive.ql.parse.SemanticException: Database does not exist: ttt at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.getDatabase(BaseSemanticAnalyzer.java:2154) at org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer.analyzeDropDatabase(DDLSemanticAnalyzer.java:1393) at org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer.analyzeInternal(DDLSemanticAnalyzer.java:514) at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:289) at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:666) at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:1881) at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:1828) at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:1823) at org.apache.hadoop.hive.ql.reexec.ReExecDriver.compileAndRespond(ReExecDriver.java:126) at org.apache.hive.service.cli.operation.SQLOperation.prepare(SQLOperation.java:197) at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork$1.run(SQLOperation.java:322) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1730) at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork.run(SQLOperation.java:342) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) +<15>1 2019-03-21T07:07:17.035Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070717_45f3313d-83e8-4994-a998-a4e60b19b136" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-92"] +<14>1 2019-03-21T07:07:17.036Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070717_45f3313d-83e8-4994-a998-a4e60b19b136" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-92"] Completed compiling command(queryId=hive_20190321070717_45f3313d-83e8-4994-a998-a4e60b19b136); Time taken: 0.026 seconds +<15>1 2019-03-21T07:07:17.036Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070717_45f3313d-83e8-4994-a998-a4e60b19b136" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-92"] +<14>1 2019-03-21T07:07:17.036Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="lockmgr.DbTxnManager" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070717_45f3313d-83e8-4994-a998-a4e60b19b136" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-92"] Stopped heartbeat for query: null +<15>1 2019-03-21T07:07:17.046Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070717_45f3313d-83e8-4994-a998-a4e60b19b136" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-92"] +<11>1 2019-03-21T07:07:17.105Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="operation.Operation" level="ERROR" operationLogLevel="EXECUTION" queryId="hive_20190321070717_45f3313d-83e8-4994-a998-a4e60b19b136" sessionId="1ba9df58-94d3-4235-8920-cc4a8c7afd84" thread="HiveServer2-Background-Pool: Thread-92"] Error running hive query: org.apache.hive.service.cli.HiveSQLException: Error while compiling statement: FAILED: SemanticException [Error 10072]: Database does not exist: ttt at org.apache.hive.service.cli.operation.Operation.toSQLException(Operation.java:348) at org.apache.hive.service.cli.operation.SQLOperation.prepare(SQLOperation.java:199) at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork$1.run(SQLOperation.java:322) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1730) at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork.run(SQLOperation.java:342) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hive.ql.parse.SemanticException: Database does not exist: ttt at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.getDatabase(BaseSemanticAnalyzer.java:2154) at org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer.analyzeDropDatabase(DDLSemanticAnalyzer.java:1393) at org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer.analyzeInternal(DDLSemanticAnalyzer.java:514) at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:289) at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:666) at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:1881) at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:1828) at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:1823) at org.apache.hadoop.hive.ql.reexec.ReExecDriver.compileAndRespond(ReExecDriver.java:126) at org.apache.hive.service.cli.operation.SQLOperation.prepare(SQLOperation.java:197) ... 12 more +<14>1 2019-03-21T07:07:17.338Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Using the default value passed in for log id: 1ba9df58-94d3-4235-8920-cc4a8c7afd84 +<14>1 2019-03-21T07:07:17.339Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Updating thread name to 1ba9df58-94d3-4235-8920-cc4a8c7afd84 HiveServer2-HttpHandler-Pool: Thread-35 +<14>1 2019-03-21T07:07:17.339Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="1ba9df58-94d3-4235-8920-cc4a8c7afd84 HiveServer2-HttpHandler-Pool: Thread-35"] Using the default value passed in for log id: 1ba9df58-94d3-4235-8920-cc4a8c7afd84 +<14>1 2019-03-21T07:07:17.339Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="1ba9df58-94d3-4235-8920-cc4a8c7afd84 HiveServer2-HttpHandler-Pool: Thread-35"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-35 +<12>1 2019-03-21T07:07:20.145Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="WARN" thread="NotificationEventPoll 0"] HiveConf of name hive.stats.fetch.partition.stats does not exist +<12>1 2019-03-21T07:07:20.145Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="WARN" thread="NotificationEventPoll 0"] HiveConf of name hive.metastore.runworker.in does not exist +<12>1 2019-03-21T07:07:20.146Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="WARN" thread="NotificationEventPoll 0"] HiveConf of name hive.heapsize does not exist +<14>1 2019-03-21T07:07:20.148Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="NotificationEventPoll 0"] Trying to connect to metastore with URI thrift://metastore-service.warehouse-1553110190-8zwt.svc.cluster.local:9083 +<14>1 2019-03-21T07:07:20.149Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="NotificationEventPoll 0"] Opened a connection to metastore, current connections: 7 +<14>1 2019-03-21T07:07:20.149Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" thread="NotificationEventPoll 0"] Connected to metastore. +<14>1 2019-03-21T07:07:20.149Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.RetryingMetaStoreClient" level="INFO" thread="NotificationEventPoll 0"] RetryingMetaStoreClient proxy=class org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient ugi=hive (auth:SIMPLE) retries=24 delay=5 lifetime=0 +<14>1 2019-03-21T07:07:52.025Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="thrift.ThriftHttpServlet" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Could not validate cookie sent, will try to generate a new cookie +<14>1 2019-03-21T07:07:52.026Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="thrift.ThriftHttpServlet" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Cookie added for clientUserName anonymous +<14>1 2019-03-21T07:07:52.026Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="thrift.ThriftCLIService" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Client protocol version: HIVE_CLI_SERVICE_PROTOCOL_V10 +<14>1 2019-03-21T07:07:52.332Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Created HDFS directory: /tmp/hive/hive/42b33e05-6bba-4a0c-a59e-b89b8b6394cc +<14>1 2019-03-21T07:07:52.336Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Created local directory: /tmp/hive/42b33e05-6bba-4a0c-a59e-b89b8b6394cc +<14>1 2019-03-21T07:07:52.595Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Created HDFS directory: /tmp/hive/hive/42b33e05-6bba-4a0c-a59e-b89b8b6394cc/_tmp_space.db +<14>1 2019-03-21T07:07:52.599Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.HiveSessionImpl" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Operation log session directory is created: /tmp/hive/operation_logs/42b33e05-6bba-4a0c-a59e-b89b8b6394cc +<14>1 2019-03-21T07:07:52.599Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="service.CompositeService" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-35"] Session opened, SessionHandle [42b33e05-6bba-4a0c-a59e-b89b8b6394cc], current sessions:3 +<14>1 2019-03-21T07:07:52.659Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-37"] Using the default value passed in for log id: 42b33e05-6bba-4a0c-a59e-b89b8b6394cc +<14>1 2019-03-21T07:07:52.660Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="HiveServer2-HttpHandler-Pool: Thread-37"] Updating thread name to 42b33e05-6bba-4a0c-a59e-b89b8b6394cc HiveServer2-HttpHandler-Pool: Thread-37 +<14>1 2019-03-21T07:07:52.661Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="operation.OperationManager" level="INFO" thread="42b33e05-6bba-4a0c-a59e-b89b8b6394cc HiveServer2-HttpHandler-Pool: Thread-37"] Adding operation: OperationHandle [opType=EXECUTE_STATEMENT, getHandleIdentifier()=264ea00f-0554-429d-8db7-7c9bc9878238] +<14>1 2019-03-21T07:07:52.661Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="conf.HiveConf" level="INFO" thread="42b33e05-6bba-4a0c-a59e-b89b8b6394cc HiveServer2-HttpHandler-Pool: Thread-37"] Using the default value passed in for log id: 42b33e05-6bba-4a0c-a59e-b89b8b6394cc +<14>1 2019-03-21T07:07:52.661Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="session.SessionState" level="INFO" thread="42b33e05-6bba-4a0c-a59e-b89b8b6394cc HiveServer2-HttpHandler-Pool: Thread-37"] Resetting thread name to HiveServer2-HttpHandler-Pool: Thread-37 +<15>1 2019-03-21T07:07:52.663Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] +<15>1 2019-03-21T07:07:52.663Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] +<15>1 2019-03-21T07:07:52.663Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] +<15>1 2019-03-21T07:07:52.663Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] +<14>1 2019-03-21T07:07:52.663Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Compiling command(queryId=hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159): repl dump `*` with ('hive.repl.dump.metadata.only'='true', 'hive.repl.dump.include.acid.tables'='true') +<15>1 2019-03-21T07:07:52.689Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] +<15>1 2019-03-21T07:07:52.691Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] +<15>1 2019-03-21T07:07:52.691Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] +<14>1 2019-03-21T07:07:52.698Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Trying to connect to metastore with URI thrift://metastore-service.warehouse-1553110190-8zwt.svc.cluster.local:9083 +<14>1 2019-03-21T07:07:52.698Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Opened a connection to metastore, current connections: 8 +<14>1 2019-03-21T07:07:52.699Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.HiveMetaStoreClient" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Connected to metastore. +<14>1 2019-03-21T07:07:52.699Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="metastore.RetryingMetaStoreClient" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] RetryingMetaStoreClient proxy=class org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient ugi=hive (auth:SIMPLE) retries=24 delay=5 lifetime=0 +<14>1 2019-03-21T07:07:52.716Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="lockmgr.DbTxnManager" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Opened txnid:670 +<14>1 2019-03-21T07:07:52.727Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="sqlstd.SQLStdHiveAccessController" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Created SQLStdHiveAccessController for session context : HiveAuthzSessionContext [sessionString=42b33e05-6bba-4a0c-a59e-b89b8b6394cc, clientType=HIVESERVER2] +<14>1 2019-03-21T07:07:52.749Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Semantic Analysis Completed (retrial = false) +<15>1 2019-03-21T07:07:52.749Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] +<14>1 2019-03-21T07:07:52.750Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:dump_dir, type:string, comment:from deserializer), FieldSchema(name:last_repl_id, type:string, comment:from deserializer)], properties:null) +<14>1 2019-03-21T07:07:52.750Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="exec.ListSinkOperator" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Initializing operator LIST_SINK[0] +<15>1 2019-03-21T07:07:52.751Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] +<15>1 2019-03-21T07:07:52.751Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] +<15>1 2019-03-21T07:07:52.751Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] +<14>1 2019-03-21T07:07:52.751Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Completed compiling command(queryId=hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159); Time taken: 0.088 seconds +<14>1 2019-03-21T07:07:52.752Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="reexec.ReExecDriver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Execution #1 of query +<15>1 2019-03-21T07:07:52.752Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] +<14>1 2019-03-21T07:07:52.752Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="lockmgr.DbTxnManager" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Setting lock request transaction to txnid:670 for queryId=hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159 +<14>1 2019-03-21T07:07:52.757Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="lockmgr.DbLockManager" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Requesting: queryId=hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159 LockRequest(component:[LockComponent(type:SHARED_READ, level:DB, dbname:airline_ontime, operationType:SELECT), LockComponent(type:SHARED_READ, level:DB, dbname:default, operationType:SELECT), LockComponent(type:SHARED_READ, level:DB, dbname:information_schema, operationType:SELECT), LockComponent(type:SHARED_READ, level:DB, dbname:sys, operationType:SELECT)], txnid:670, user:hive, hostname:hiveserver2-99ddccbd8-49sbv, agentInfo:hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159) +<14>1 2019-03-21T07:07:52.773Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="lockmgr.DbLockManager" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Response to queryId=hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159 LockResponse(lockid:646, state:ACQUIRED) +<15>1 2019-03-21T07:07:52.774Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] +<15>1 2019-03-21T07:07:52.776Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] +<14>1 2019-03-21T07:07:52.776Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Executing command(queryId=hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159): repl dump `*` with ('hive.repl.dump.metadata.only'='true', 'hive.repl.dump.include.acid.tables'='true') +<15>1 2019-03-21T07:07:52.776Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] +<15>1 2019-03-21T07:07:52.776Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] +<15>1 2019-03-21T07:07:52.776Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] +<15>1 2019-03-21T07:07:52.776Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] +<15>1 2019-03-21T07:07:52.777Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="log.PerfLogger" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] +<14>1 2019-03-21T07:07:52.777Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ql.Driver" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Starting task [Stage-0:REPL_DUMP] in serial mode +<14>1 2019-03-21T07:07:52.803Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ReplState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] REPL::START: {"dbName":"airline_ontime","dumpType":"BOOTSTRAP","estimatedNumTables":4,"estimatedNumFunctions":0,"dumpStartTime":1553152072} +<14>1 2019-03-21T07:07:53.350Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="dump.Utils" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] REPL DUMP:: Set property for Database: airline_ontime, Property: bootstrap.dump.state.94ab1139-bc1c-4fff-beae-6ec53dd30823, Value: ACTIVE +<14>1 2019-03-21T07:07:53.431Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="parse.EximUtil" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Path before norm :/clusters/cluster-1547937522-w74v/warehouse-1553110190-8zwt/user/hive/repl/7decaaea-a0a9-4cd6-8b86-f1cbab269d6a/airline_ontime/flights +<14>1 2019-03-21T07:07:53.432Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="parse.EximUtil" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Scheme:s3a, authority:edws-llap-perf, path:/clusters/cluster-1547937522-w74v/warehouse-1553110190-8zwt/user/hive/repl/7decaaea-a0a9-4cd6-8b86-f1cbab269d6a/airline_ontime/flights +<14>1 2019-03-21T07:07:54.297Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ReplState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] REPL::TABLE_DUMP: {"dbName":"airline_ontime","tableName":"flights","tableType":"MANAGED_TABLE","tablesDumpProgress":"1/4","dumpTime":1553152074} +<14>1 2019-03-21T07:07:54.351Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="parse.EximUtil" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Path before norm :/clusters/cluster-1547937522-w74v/warehouse-1553110190-8zwt/user/hive/repl/7decaaea-a0a9-4cd6-8b86-f1cbab269d6a/airline_ontime/airports +<14>1 2019-03-21T07:07:54.351Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="parse.EximUtil" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Scheme:s3a, authority:edws-llap-perf, path:/clusters/cluster-1547937522-w74v/warehouse-1553110190-8zwt/user/hive/repl/7decaaea-a0a9-4cd6-8b86-f1cbab269d6a/airline_ontime/airports +<14>1 2019-03-21T07:07:56.022Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ReplState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] REPL::TABLE_DUMP: {"dbName":"airline_ontime","tableName":"airports","tableType":"MANAGED_TABLE","tablesDumpProgress":"2/4","dumpTime":1553152076} +<14>1 2019-03-21T07:07:56.096Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="parse.EximUtil" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Path before norm :/clusters/cluster-1547937522-w74v/warehouse-1553110190-8zwt/user/hive/repl/7decaaea-a0a9-4cd6-8b86-f1cbab269d6a/airline_ontime/airlines +<14>1 2019-03-21T07:07:56.096Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="parse.EximUtil" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Scheme:s3a, authority:edws-llap-perf, path:/clusters/cluster-1547937522-w74v/warehouse-1553110190-8zwt/user/hive/repl/7decaaea-a0a9-4cd6-8b86-f1cbab269d6a/airline_ontime/airlines +<14>1 2019-03-21T07:07:56.835Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ReplState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] REPL::TABLE_DUMP: {"dbName":"airline_ontime","tableName":"airlines","tableType":"MANAGED_TABLE","tablesDumpProgress":"3/4","dumpTime":1553152076} +<14>1 2019-03-21T07:07:56.861Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="parse.EximUtil" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Path before norm :/clusters/cluster-1547937522-w74v/warehouse-1553110190-8zwt/user/hive/repl/7decaaea-a0a9-4cd6-8b86-f1cbab269d6a/airline_ontime/planes +<14>1 2019-03-21T07:07:56.861Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="parse.EximUtil" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Scheme:s3a, authority:edws-llap-perf, path:/clusters/cluster-1547937522-w74v/warehouse-1553110190-8zwt/user/hive/repl/7decaaea-a0a9-4cd6-8b86-f1cbab269d6a/airline_ontime/planes +<14>1 2019-03-21T07:07:57.519Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ReplState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] REPL::TABLE_DUMP: {"dbName":"airline_ontime","tableName":"planes","tableType":"MANAGED_TABLE","tablesDumpProgress":"4/4","dumpTime":1553152077} +<14>1 2019-03-21T07:07:57.629Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="dump.Utils" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] REPL DUMP:: Reset property for Database: airline_ontime, Property: bootstrap.dump.state.94ab1139-bc1c-4fff-beae-6ec53dd30823 +<14>1 2019-03-21T07:07:57.631Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ReplState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] REPL::END: {"dbName":"airline_ontime","dumpType":"BOOTSTRAP","actualNumTables":4,"actualNumFunctions":0,"dumpEndTime":1553152077,"dumpDir":"/clusters/cluster-1547937522-w74v/warehouse-1553110190-8zwt/user/hive/repl/7decaaea-a0a9-4cd6-8b86-f1cbab269d6a","lastReplId":"1535"} +<14>1 2019-03-21T07:07:57.636Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ReplState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] REPL::START: {"dbName":"default","dumpType":"BOOTSTRAP","estimatedNumTables":0,"estimatedNumFunctions":0,"dumpStartTime":1553152077} +<14>1 2019-03-21T07:07:58.038Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="dump.Utils" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] REPL DUMP:: Set property for Database: default, Property: bootstrap.dump.state.6d62a7d2-aa5d-4506-a9d6-98e36ac2a032, Value: ACTIVE +<14>1 2019-03-21T07:07:58.135Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="dump.Utils" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] REPL DUMP:: Reset property for Database: default, Property: bootstrap.dump.state.6d62a7d2-aa5d-4506-a9d6-98e36ac2a032 +<14>1 2019-03-21T07:07:58.135Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ReplState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] REPL::END: {"dbName":"default","dumpType":"BOOTSTRAP","actualNumTables":0,"actualNumFunctions":0,"dumpEndTime":1553152078,"dumpDir":"/clusters/cluster-1547937522-w74v/warehouse-1553110190-8zwt/user/hive/repl/7decaaea-a0a9-4cd6-8b86-f1cbab269d6a","lastReplId":"1535"} +<14>1 2019-03-21T07:07:58.140Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ReplState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] REPL::START: {"dbName":"information_schema","dumpType":"BOOTSTRAP","estimatedNumTables":6,"estimatedNumFunctions":0,"dumpStartTime":1553152078} +<14>1 2019-03-21T07:07:58.526Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="dump.Utils" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] REPL DUMP:: Set property for Database: information_schema, Property: bootstrap.dump.state.77a7e161-8f21-4262-b75d-db1d5a9a4e03, Value: ACTIVE +<14>1 2019-03-21T07:07:58.542Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="parse.EximUtil" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Path before norm :/clusters/cluster-1547937522-w74v/warehouse-1553110190-8zwt/user/hive/repl/7decaaea-a0a9-4cd6-8b86-f1cbab269d6a/information_schema/columns +<14>1 2019-03-21T07:07:58.542Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="parse.EximUtil" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Scheme:s3a, authority:edws-llap-perf, path:/clusters/cluster-1547937522-w74v/warehouse-1553110190-8zwt/user/hive/repl/7decaaea-a0a9-4cd6-8b86-f1cbab269d6a/information_schema/columns +<14>1 2019-03-21T07:07:59.156Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="ReplState" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] REPL::TABLE_DUMP: {"dbName":"information_schema","tableName":"columns","tableType":"VIRTUAL_VIEW","tablesDumpProgress":"1/6","dumpTime":1553152079} +<14>1 2019-03-21T07:07:59.223Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="parse.EximUtil" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Path before norm :/clusters/cluster-1547937522-w74v/warehouse-1553110190-8zwt/user/hive/repl/7decaaea-a0a9-4cd6-8b86-f1cbab269d6a/information_schema/schemata +<14>1 2019-03-21T07:07:59.223Z hiveserver2-99ddccbd8-49sbv hiveserver2 1 cb6c1a29-4ba7-11e9-83d0-02c17ec35ac0 [mdc@18060 class="parse.EximUtil" level="INFO" operationLogLevel="EXECUTION" queryId="hive_20190321070752_2cff398a-e533-4dee-816b-f7683c82f159" sessionId="42b33e05-6bba-4a0c-a59e-b89b8b6394cc" thread="HiveServer2-Background-Pool: Thread-94"] Scheme:s3a, authority:edws-llap-perf, path:/clusters/cluster-1547937522-w74v/warehouse-1553110190-8zwt/user/hive/repl/7decaaea-a0a9-4cd6-8b86-f1cbab269d6a/information_schema/schemata + diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 96dfbc4b56..c0ef1bbec1 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -213,6 +213,7 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\ stats_only_null.q,\ subquery_exists.q,\ subquery_in.q,\ + syslog_parser.q,\ temp_table.q,\ topnkey.q,\ union2.q,\ diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java index 9ef7af4eb0..91c94efaf5 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java @@ -612,8 +612,8 @@ public IncludesImpl(List tableIncludedCols, boolean isAcidScan, // currently implemented in Hive). The compile should supply the columns it expects // to see, which is not "all, of any schema". Is VRB row CVs the right mechanism // for that? Who knows. Perhaps resolve in schema evolution v2. - tableIncludedCols = new ArrayList<>(rbCtx.getRowColumnTypeInfos().length); - for (int i = 0; i < rbCtx.getRowColumnTypeInfos().length; ++i) { + tableIncludedCols = new ArrayList<>(rbCtx.getDataColumnCount()); + for (int i = 0; i < rbCtx.getDataColumnCount(); ++i) { tableIncludedCols.add(i); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java b/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java index 27fe828b75..bfabdce736 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java @@ -204,6 +204,9 @@ private static Object boxLiteral(ExprNodeConstantDesc constantDesc, } return fl.doubleValue(); case TIMESTAMP: + if (lit instanceof org.apache.hadoop.hive.common.type.Timestamp) { + return ((org.apache.hadoop.hive.common.type.Timestamp) lit).toSqlTimestamp(); + } return Timestamp.valueOf(lit.toString()); case DATE: return Date.valueOf(lit.toString()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/syslog/SyslogInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/log/syslog/SyslogInputFormat.java new file mode 100644 index 0000000000..10f8c10b9a --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/log/syslog/SyslogInputFormat.java @@ -0,0 +1,277 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.log.syslog; + +import java.io.IOException; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.GregorianCalendar; +import java.util.List; +import java.util.Locale; +import java.util.TimeZone; +import java.util.concurrent.TimeUnit; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.exec.SerializationUtilities; +import org.apache.hadoop.hive.ql.io.parquet.ProjectionPusher; +import org.apache.hadoop.hive.ql.io.sarg.ConvertAstToSearchArg; +import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; +import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; +import org.apache.hadoop.hive.ql.plan.TableScanDesc; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.FileSplit; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordReader; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.TextInputFormat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.VisibleForTesting; + +/** + * Syslog input format handles files written to sys.logs hive table. This input format can prune files based on + * predicates in ts (timestamp) column. Following are the assumptions/supported options for the file pruning to work. + * When configuring log shippers (like fluentd), the following directory layout has to be followed + * + * WAREHOUSE_DIR/sys.db/logs/dt=DATE/ns=NAMESPACE/app=APPLICATION/YYYY-mm-DD-HH-MM_i[.log][.gz] + * WAREHOUSE_DIR - is hive warehouse location for external tables + * DATE - date of the log files + * NAMESPACE - namespace for the applications + * APPLICATION - name of the application (hiveserver2, metastore etc.) + * YYYY - year + * mm - month + * DD - day of the month + * HH - hour of the day (24 hr format) + * MM - minute + * i - index (log rotation) + * [.log] - optional log extension + * [.gz] - optional gz compression extension + * + * When executing queries like + * select count(*) from sys.logs where ts between current_timestamp() - INTERVAL '1' DAY and current_timestamp() + * + * syslog input format can prunes the file listing based on the query filter condition. + * File Pruning Assumptions/Support: + * - All timestamps are in UTC (timestamp in file name, log lines and in query predicates all assume UTC) + * - File names are in YYYY-mm-DD-HH-MM_i[.log][.gz] format. + * - Seconds and millis in predicates gets rounded off to nearest minute (essentially ignoring them). + * - Time slice used by log aggregation library should match with hive.syslog.input.format.file.time.slice config. + * If a filename is 2019-04-02-21-00_0.log.gz and timeslice is 300s then the file 2019-04-02-21-00_0.log.gz is + * expected to have log lines from timestamp 2019:04:02 21:00:00 to 2019:04:02 21:05:00 timestamp. + * - Logs table should have 'ts' as timestamp column. + * - Only simple BETWEEN filter predicate is supported for 'ts' column. There cannot be >1 predicates on 'ts' column. + */ +public class SyslogInputFormat extends TextInputFormat { + private static final Logger LOG = LoggerFactory.getLogger(SyslogInputFormat.class); + private static final long MILLISECONDS_PER_MINUTE = 60 * 1000L; + private SearchArgument sarg; + private JobConf jobConf; + private ProjectionPusher projectionPusher = new ProjectionPusher(); + + enum Location { + BEFORE, + MIN, + MIDDLE, + MAX, + AFTER + } + + @Override + protected FileStatus[] listStatus(final JobConf job) throws IOException { + FileStatus[] fileStatuses = super.listStatus(job); + String filterExprSerialized = job.get(TableScanDesc.FILTER_EXPR_CONF_STR); + final boolean filePruning = HiveConf.getBoolVar(job, HiveConf.ConfVars.SYSLOG_INPUT_FORMAT_FILE_PRUNING); + final long timeSliceSeconds = HiveConf.getTimeVar(job, HiveConf.ConfVars.SYSLOG_INPUT_FORMAT_FILE_TIME_SLICE, + TimeUnit.SECONDS); + if (filePruning && filterExprSerialized != null) { + ExprNodeGenericFuncDesc filterExpr = SerializationUtilities.deserializeExpression(filterExprSerialized); + // we receive the expression filters from hive and convert it to sarg keeping only timestamp (ts) column from + // sys.logs table. The SARG on ts column is evaluated against the files that are formatted in YYYY-mm-dd-MM_* format + this.sarg = ConvertAstToSearchArg.create(job, filterExpr); + } + LOG.info("Syslog configs - filePruning: {} SARG: {} timeSliceSeconds: {}", filePruning, sarg, timeSliceSeconds); + if (sarg != null) { + fileStatuses = pruneFiles(sarg, timeSliceSeconds, Arrays.asList(fileStatuses)).toArray(new FileStatus[0]); + } + return fileStatuses; + } + + @VisibleForTesting + public List pruneFiles(final SearchArgument sarg, final long timeSliceSeconds, + final List inputFiles) { + int tsExpr = 0; + List predLeaves = sarg.getLeaves(); + SearchArgument.TruthValue[] truthValues = new SearchArgument.TruthValue[predLeaves.size()]; + PredicateLeaf tsPred = null; + for (int i = 0; i < truthValues.length; i++) { + if (!predLeaves.get(i).getColumnName().equalsIgnoreCase("ts")) { + truthValues[i] = SearchArgument.TruthValue.YES_NO_NULL; + } else { + tsPred = predLeaves.get(i); + tsExpr++; + } + } + // multiple expressions on 'ts' columns are not supported + if (tsExpr == 0 || tsExpr > 1) { + // no expressions on timestamp column or multi-expression + if (tsExpr == 0) { + LOG.warn("No filter expression on 'ts' column. Skipping file pruning.."); + } else { + LOG.warn("Multi-filter expression ({}) on 'ts' column is not supported. Skipping file pruning..", tsExpr); + } + return inputFiles; + } + + // only BETWEEN filter is supported + if (tsPred.getOperator() == PredicateLeaf.Operator.BETWEEN) { + List selectedFiles = new ArrayList<>(); + List objects = tsPred.getLiteralList(); + + Timestamp left = (Timestamp) objects.get(0); + Timestamp right = (Timestamp) objects.get(1); + // file pruning does not handle seconds and millis granularity. + // so we round to nearest minute (left = floor) and (right = ceil). + Timestamp predLowerBound = roundupToMinuteFloor(left); + Timestamp predUpperBound = roundupToMinuteCeil(right); + + // we have left and right values from BETWEEN expression. For each file we have a start timestamp (from filename) + // and end timestamp (start + timeslice). We compare the predicates with start and end timestamp of each file + // and see if it falls within the range. + for (FileStatus fileStatus : inputFiles) { + Timestamp fileLowerBound = getTimeStampFromPath(fileStatus); + if (fileLowerBound != null) { + Timestamp fileUpperBound = Timestamp.from(fileLowerBound.toInstant().plusSeconds(timeSliceSeconds)); + Location predLowerLocation; + Location predUpperLocation = null; + predLowerLocation = compareToRange(predLowerBound, fileLowerBound, fileUpperBound); + boolean selected = false; + if (predLowerLocation == Location.BEFORE || predLowerLocation == Location.MIN) { + predUpperLocation = compareToRange(predUpperBound, fileLowerBound, fileUpperBound); + if (predUpperLocation == Location.AFTER || predUpperLocation == Location.MAX) { + selectedFiles.add(fileStatus); + selected = true; + } else if (predUpperLocation == Location.BEFORE) { + // file skipped (out of range) + } else { + selectedFiles.add(fileStatus); + selected = true; + } + } else if (predLowerLocation == Location.AFTER) { + // file skipped (out of range) + } else { + selectedFiles.add(fileStatus); + selected = true; + } + + LOG.info("file: {} -> [{}, {}] against predicate [{}({}), {}({})]. selected? {}", + fileStatus.getPath(), fileLowerBound.toInstant(), fileUpperBound.toInstant(), + predLowerBound.toInstant(), predLowerLocation, predUpperBound.toInstant(), predUpperLocation, selected); + } else { + LOG.warn("Timestamp cannot be extracted from filename. Incorrect file name convention? {}", + fileStatus.getPath()); + return inputFiles; + } + } + LOG.info("Total selected files: {}", selectedFiles.size()); + return selectedFiles; + } + LOG.warn("Unsupported expression ({}) on 'ts' column. Skipping file pruning..", tsPred.getOperator()); + return inputFiles; + } + + // 2019-10-13 10:15:35.100 -> 2019-10-13 10:15:00.000 + private Timestamp roundupToMinuteFloor(final Timestamp ts) { + long millis = ts.getTime(); + long newMillis = MILLISECONDS_PER_MINUTE * (millis / MILLISECONDS_PER_MINUTE); + return new Timestamp(newMillis); + } + + // 2019-10-13 10:15:35.100 -> 2019-10-13 10:16:00.000 + private Timestamp roundupToMinuteCeil(final Timestamp ts) { + long millis = ts.getTime(); + long newMillis = MILLISECONDS_PER_MINUTE * (millis / MILLISECONDS_PER_MINUTE); + newMillis += MILLISECONDS_PER_MINUTE; + return new Timestamp(newMillis); + } + + // 2019-04-02-21-00_0.log.gz -> 2019:04:02 21:00:00.0 + private static Timestamp getTimeStampFromPath(final FileStatus fileStatus) { + String file = fileStatus.getPath().getName(); + if (file.contains("_")) { + String tsStr = file.split("_")[0]; + if (tsStr.contains("-")) { + String[] tsTokens = tsStr.split("-"); + if (tsTokens.length == 5) { + try { + Calendar calendar = new GregorianCalendar(TimeZone.getTimeZone("UTC"), Locale.getDefault()); + int year = Integer.parseUnsignedInt(tsTokens[0]); + int month = Integer.parseUnsignedInt(tsTokens[1]); + int day = Integer.parseUnsignedInt(tsTokens[2]); + int hour = Integer.parseUnsignedInt(tsTokens[3]); + int min = Integer.parseUnsignedInt(tsTokens[4]); + // month value is zero indexed + calendar.set(year, month - 1, day, hour, min, 0); + Timestamp ts = new Timestamp(calendar.getTimeInMillis()); + ts.setNanos(0); + return ts; + } catch (NumberFormatException e) { + return null; + } + } + } + } + return null; + } + + private Location compareToRange(Comparable point, T min, T max) { + int minCompare = point.compareTo(min); + if (minCompare < 0) { + return Location.BEFORE; + } else if (minCompare == 0) { + return Location.MIN; + } else { + int maxCompare = point.compareTo(max); + if (maxCompare > 0) { + return Location.AFTER; + } else { + return maxCompare == 0 ? Location.MAX : Location.MIDDLE; + } + } + } + + @Override + public RecordReader getRecordReader(final InputSplit genericSplit, final JobConf job, + final Reporter reporter) + throws IOException { + if (genericSplit instanceof FileSplit) { + final Path finalPath = ((FileSplit) genericSplit).getPath(); + LOG.debug("Returning record reader for path {}", finalPath); + jobConf = projectionPusher.pushProjectionsAndFilters(job, finalPath.getParent()); + } + return super.getRecordReader(genericSplit, jobConf, reporter); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/syslog/SyslogParser.java b/ql/src/java/org/apache/hadoop/hive/ql/log/syslog/SyslogParser.java new file mode 100644 index 0000000000..2852e53a4e --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/log/syslog/SyslogParser.java @@ -0,0 +1,790 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.log.syslog; + +import java.io.ByteArrayOutputStream; +import java.io.Closeable; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.GregorianCalendar; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.TimeZone; + +import org.apache.commons.lang3.StringEscapeUtils; +import org.apache.hadoop.hive.common.type.Timestamp; + +/** + * A Syslog protocol parser. + * It should be capable of parsing RFC 3164 (BSD syslog) streams as well as RFC 5424 (defined in 2009.). + * Adapted from https://github.com/spotify/flume-syslog-source2 and modified it for several assumptions + * about the way hive logs using syslog format (specifically RFC5424). + * + * This implementation also parses structured data, returns all parsed fields as map and also un-escapes messages. + * This parser also gracefully handles some corner cases where 'msg' can be empty or line can start with '<' but not + * a valid RFC5424 format etc. + * + * Assumption: + * 1) This parser assumes the linebreaks '\n' in stack traces for example are replaced by '\r' to make single + * line message. The reader will do replacement of '\r' with '\n' at the time of read. + * 2) This parser assumes structured data values are html escaped. So it will html unescape when parsing structured + * data. (hive writes log lines directly to stderr that look like rfc5424 layout starting with '<' so the expectation + * from log4j2 is to escape those lines using html escaping). + * 3) Read event returns List conforming to sys.logs table schema in hive. The schema for sys.logs table is + * expected to be (facility STRING, severity STRING, version STRING, ts TIMESTAMP, hostname STRING, app_name STRING, + * proc_id STRING, msg_id STRING, structured_data map, msg BINARY, unmatched BINARY) + * 4) Timestamps are in UTC + * + * This parser is tested with Log4j2's RFC5424 layout generated using the following properties + * appenders = console + * appender.console.layout.type = Rfc5424Layout + * appender.console.layout.appName = ${env:APP_NAME} + * appender.console.layout.facility = USER + * appender.console.layout.includeMDC = true + * appender.console.layout.mdcId = mdc + * appender.console.layout.messageId = ${env:MSG_ID} + * appender.console.layout.newLine = true + * appender.console.layout.newLineEscape = \\r + * appender.console.layout.exceptionPattern = %ex{full} + * appender.console.layout.loggerfields.type = LoggerFields + * appender.console.layout.loggerfields.pairs1.type = KeyValuePair + * appender.console.layout.loggerfields.pairs1.key = level + * appender.console.layout.loggerfields.pairs1.value = %p + * appender.console.layout.loggerfields.pairs2.type = KeyValuePair + * appender.console.layout.loggerfields.pairs2.key = thread + * appender.console.layout.loggerfields.pairs2.value = %enc{%t} + * appender.console.layout.loggerfields.pairs3.type = KeyValuePair + * appender.console.layout.loggerfields.pairs3.key = class + * appender.console.layout.loggerfields.pairs3.value = %c{2} + */ +public class SyslogParser implements Closeable { + // RFC 5424 section 6. + + // SYSLOG-MSG format + // PRI VERSION SP TIMESTAMP SP HOSTNAME SP APP-NAME SP PROCID SP MSGID SP STRUCTURED-DATA [SP MSG] + + // facility + severity forms PRI + // Version 0 is the RFC 3164 format, 1 for RFC 5424 format. + + // Read event returns the following schema + // facility STRING + // severity STRING + // version STRING + // ts TIMESTAMP + // hostname STRING + // app_name STRING + // proc_id STRING + // msg_id STRING + // structured_data map + // msg BINARY + // unmatched BINARY + private final static int EXPECTED_COLUMNS = 11; + + private final static String[] FACILITIES = new String[]{"KERN", "USER", "MAIL", "DAEMON", "AUTH", "SYSLOG", "LPR", "NEWS", + "UUCP", "CRON", "AUTHPRIV", "FTP", "NTP", "AUDIT", "ALERT", "CLOCK", "LOCAL0", "LOCAL1", "LOCAL2", "LOCAL3", + "LOCAL4", "LOCAL5", "LOCAL6", "LOCAL7"}; + + // As defined in RFC 5424. + private final static int MAX_SUPPORTED_VERSION = 1; + + private InputStream in; + private boolean parseTag; + private static final Charset UTF8 = StandardCharsets.UTF_8; + private Charset charset; + + /// Push back buffer. -1 indicates that it is empty. + private int pushBack = -1; + + + public SyslogParser() { + this(null); + } + + /** + * Construct a new Syslog protocol parser. + * Tags are parsed, and the encoding is assumed to be UTF-8. + */ + public SyslogParser(InputStream in) { + this(in, true, UTF8); + } + + /** + * Construct a new Syslog protocol parser. + * + * @param in the stream to read data from. The InputStream#read() + * function is heavily used, so make sure it is buffered. + * @param parseTag true to parse the "tag[pid]:", false to leave it as + * part of the message body. + * @param encoding the encoding to use for various string conversions, + * most notably the hostname. + */ + public SyslogParser(InputStream in, boolean parseTag, Charset encoding) { + this.in = in; + this.parseTag = parseTag; + this.charset = encoding; + } + + /** + * Free the resources used by this parser. + * Note that the parser cannot be reused. Closes the underlying input + * stream. + */ + public void close() throws IOException { + in.close(); + } + + /** + * Read the next Syslog message from the stream. + * + * @return a parsed map of object, or null on EOF. + * @throw EOFException if EOF is found in an inappropriate place. + * @throw IOException if the underlying stream fails, or unexpected + * bytes are seen. + */ + public List readEvent() throws IOException { + int priority; + int c = read(false); + List row = new ArrayList<>(EXPECTED_COLUMNS); + // Return null on initial EOF. + if (c == -1) { + return null; + } + + if (c == '<') { + priority = readInt(); + if (priority == -1) { + return unmatchedEvent(c); + } + expect('>'); + } else { + return unmatchedEvent(c); + } + + int version = 0; + Calendar cal; + + if (Character.isDigit(peek())) { + // Assume ISO date and time + int y = readInt(); + + c = read(true); + + if (c == ' ') { + // Assume this is a RFC 5424 message. + version = y; + + if (version > MAX_SUPPORTED_VERSION) { + throw new IOException("Unsupported syslog version: " + version); + } + + skipSpaces(); + y = readInt(); + expect('-'); + } else if (c != '-') { + throw new IOException("Unexpected syslog character: " + (char) c); + } + + int m = readInt(); + expect('-'); + int d = readInt(); + + c = read(true); + + if (c != 'T' && c != ' ') { + throw new IOException("Unexpected syslog character: " + (char) c); + } + + int hh = readInt(); + expect(':'); + int mm = readInt(); + expect(':'); + int ss = readInt(); + double subss = 0; + + c = read(true); + + if (c == '.') { + // Fractions of seconds + subss = readFractions(); + c = read(true); + } + + int tz = 0; + + if (c == 'Z') { + // UTC + } else if (c == '-') { + tz = readInt(); + + if (peek() == ':') { + read(true); + tz = -(tz * 60 + readInt()); + } + } else if (c == '+') { + tz = readInt(); + + if (peek() == ':') { + read(true); + tz = tz * 60 + readInt(); + } + } + + cal = new GregorianCalendar(TimeZone.getTimeZone("UTC"), Locale.getDefault()); + + cal.set(y, m - 1, d, hh, mm, ss); + cal.set(Calendar.MILLISECOND, (int) (subss * 1000)); + cal.add(Calendar.MINUTE, tz); + } else { + // Assume BSD date and time + int m = readMonthAbbreviation(); + + expect(' '); + skipSpaces(); + + int d = readInt(); + + expect(' '); + skipSpaces(); + + int hh = readInt(); + + expect(':'); + + int mm = readInt(); + + expect(':'); + + int ss = readInt(); + + cal = new GregorianCalendar(Locale.ROOT); + + cal.set(Calendar.MONTH, m); + cal.set(Calendar.DAY_OF_MONTH, d); + cal.set(Calendar.HOUR_OF_DAY, hh); + cal.set(Calendar.MINUTE, mm); + cal.set(Calendar.SECOND, ss); + } + + expect(' '); + skipSpaces(); + + String hostname = readWordString(); + + expect(' '); + + byte[] appname = null; + byte[] procId = null; + byte[] msgId = null; + Map structuredData = null; + + if (version >= 1) { + appname = readWordOrNil(48); + expect(' '); + procId = readWordOrNil(12); + expect(' '); + msgId = readWordOrNil(32); + expect(' '); + structuredData = readAndParseStructuredData(); + } else if (version == 0 && parseTag) { + // Try to find a colon terminated tag. + appname = readTag(); + if (peek() == '[') { + procId = readPid(); + } + expect(':'); + } + + c = skipSpaces(); + + byte[] msg = null; + if (c != -1) { + msg = readLine(); + } + createEvent(version, priority, cal, hostname, appname, procId, msgId, structuredData, msg, row); + return row; + } + + private List unmatchedEvent(int c) throws IOException { + List row = new ArrayList<>(EXPECTED_COLUMNS); + byte[] msg = readLine(); + for (int i = 0; i < SyslogParser.EXPECTED_COLUMNS; i++) { + row.add(null); + } + row.set(10, ((char) c + new String(msg)).getBytes(charset)); + return row; + } + + /** + * Create a log event from the given parameters. + * + * @param version the syslog version, 0 for RFC 3164 + * @param priority the syslog priority, according to RFC 5424 + * @param cal the timestamp of the message. Note that timezone matters + * @param hostname the hostname + * @param appname the RFC 5424 appname + * @param procId the RFC 5424 proc-id + * @param msgId the RFC 5424 msg-id + * @param structuredData the RFC 5424 structured-data + * @param body the message body + */ + private void createEvent(int version, int priority, Calendar cal, String hostname, + byte[] appname, byte[] procId, byte[] msgId, Map structuredData, byte[] body, List row) { + row.add(FACILITIES[priority / 8]); + row.add(getEventPriorityBySyslog(priority)); + row.add(version == 0 ? "RFC3164" : "RFC5424"); + row.add(Timestamp.ofEpochMilli(cal.getTimeInMillis())); + row.add(hostname); + row.add(new String(appname)); + row.add(new String(procId)); + row.add(new String(msgId)); + row.add(structuredData); + row.add(body); + } + + /** + * Resolve the given syslog priority as a log priority. + */ + private String getEventPriorityBySyslog(int priority) { + switch (priority % 8) { + case 0: + case 1: + case 2: + return "FATAL"; + case 3: + return "ERROR"; + case 4: + return "WARN"; + case 5: + case 6: + return "INFO"; + case 7: + return "DEBUG"; + default: + // If this happens, we should tell the world. + throw new RuntimeException("Failed to look up Syslog priority"); + } + } + + /** + * Read a month value as an English abbreviation. + * see RFC 3164, Sec. 4.1.2. + */ + private int readMonthAbbreviation() throws IOException { + switch (read(true)) { + case 'A': + switch (read(true)) { + case 'p': + skipWord(); + return Calendar.APRIL; + + case 'u': + skipWord(); + return Calendar.AUGUST; + + default: + return -1; + } + + case 'D': + skipWord(); + return Calendar.DECEMBER; + + case 'F': + skipWord(); + return Calendar.FEBRUARY; + + case 'J': + read(true); // Second letter is ambiguous. + read(true); // Third letter is also ambiguous. + + switch (read(true)) { + case 'e': + skipWord(); + return Calendar.JUNE; + + case 'u': + skipWord(); + return Calendar.JANUARY; + + case 'y': + skipWord(); + return Calendar.JULY; + + default: + return -1; + } + + case 'M': + read(true); // Second letter is ambiguous. + + switch (read(true)) { + case 'r': + skipWord(); + return Calendar.MARCH; + + case 'y': + skipWord(); + return Calendar.MAY; + + default: + return -1; + } + + case 'N': + skipWord(); + return Calendar.NOVEMBER; + + case 'O': + skipWord(); + return Calendar.OCTOBER; + + case 'S': + skipWord(); + return Calendar.SEPTEMBER; + + default: + return -1; + } + } + + /** + * Read a byte and assert the value. + * + * @throw IOException if the character was unexpected + */ + private void expect(int c) throws IOException { + int d = read(true); + + if (d != c) { + throw new IOException("Unexpected syslog character: " + (char) d); + } + } + + /** + * Read until a non-whitespace ASCII byte is seen. + */ + private int skipSpaces() throws IOException { + int c; + + while ((c = read(false)) == ' ') { + continue; + } + + unread(c); + return c; + } + + /** + * Read the next byte, but then unread it again. + */ + private int peek() throws IOException { + int c = read(true); + + unread(c); + + return c; + } + + /** + * Read the next byte. + * + * @param checkEof true to throw EOFException on EOF, false to return -1. + * @return the byte, or -1 on EOF. + */ + private int read(boolean checkEof) throws IOException { + if (pushBack != -1) { + int c = pushBack; + pushBack = -1; + return c; + } + + int c = in.read(); + + if (checkEof && c == -1) { + throw new EOFException("Unexpected end of syslog stream"); + } + + return c; + } + + /** + * Push back a character. + * Only a single character can be pushed back simultaneously. + */ + private void unread(int c) { + assert c != -1 : "Trying to push back EOF"; + assert pushBack == -1 : "Trying to push back two bytes"; + pushBack = c; + } + + /** + * Read a positive integer and convert it from decimal text form. + * EOF silently terminates the number. + */ + private int readInt() throws IOException { + int c; + int ret = 0; + + boolean foundDigit = false; + while (Character.isDigit(c = read(false))) { + foundDigit = true; + ret = ret * 10 + (c - '0'); + } + + if (!foundDigit) { + unread(c); + return -1; + } + if (c != -1) { + unread(c); + } + + return ret; + } + + /** + * Read fractions (digits after a decimal point.) + * + * @return a value in the range [0, 1). + */ + private double readFractions() throws IOException { + int c; + int ret = 0; + int order = 1; + + while (Character.isDigit(c = read(false))) { + ret = ret * 10 + (c - '0'); + order *= 10; + } + + if (c != -1) { + unread(c); + } + + return (double) ret / order; + } + + /** + * Read until EOF or a space. + * The input is discarded. + */ + private void skipWord() throws IOException { + int c; + + do { + c = read(false); + } while (c != ' ' && c != -1); + + if (c != -1) { + unread(c); + } + } + + /** + * Read a word into the given output stream. + * Usually the output stream will be a ByteArrayOutputStream. + */ + private void readWord(OutputStream out) throws IOException { + int c; + + while ((c = read(false)) != ' ' && c != -1) { + out.write(c); + } + + if (c != -1) { + unread(c); + } + } + + /** + * Read a word (until next ASCII space or EOF) as a string. + * The encoding chosen while constructing the parser is used for decoding. + * + * @return a valid, but perhaps empty, word. + */ + private String readWordString() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(256); + readWord(out); + + return out.toString(charset.name()); + } + + /** + * Read a word (until next ASCII space or EOF) as a byte array. + * + * @param sizeHint an guess on how large string will be, in bytes. + * @return a valid, but perhaps empty, word. + */ + private byte[] readWord(int sizeHint) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(sizeHint); + readWord(out); + + return out.toByteArray(); + } + + /** + * Read a word (until next space or EOF) as a byte array. + * If the complete word is "-", return null. + * + * @param sizeHint an guess on how large string will be, in bytes. + */ + private byte[] readWordOrNil(int sizeHint) throws IOException { + byte[] ret = readWord(sizeHint); + + if (ret.length == 1 && ret[0] == '-') { + return null; + } + + return ret; + } + + /** + * Read a line (until next ASCII NL or EOF) as a byte array. + */ + private byte[] readLine() throws IOException { + ByteArrayOutputStream ret = new ByteArrayOutputStream(1024); + int c; + + while ((c = read(false)) != '\n' && c != -1) { + if (c != '\r') { + ret.write(c); + } else { + // CR replaced with LF + ret.write('\n'); + } + } + + return ret.toByteArray(); + } + + /** + * Read a RFC 3164 tag. + * Tags end with left bracket, colon, ASCII CR, or ASCII NL. + */ + private byte[] readTag() throws IOException { + ByteArrayOutputStream ret = new ByteArrayOutputStream(16); + int c; + + while ((c = read(true)) != ':' && c != '[' && c != '\r' && c != '\n') { + ret.write(c); + } + + unread(c); + + return ret.toByteArray(); + } + + /** + * Read a RFC 3164 pid. + * The format is "[1234]". + */ + private byte[] readPid() throws IOException { + ByteArrayOutputStream ret = new ByteArrayOutputStream(8); + int c; + + expect('['); + + while ((c = read(true)) != ']' && c != '\r' && c != '\n') { + ret.write(c); + } + + return ret.toByteArray(); + } + + /** + * Read RFC 5424 structured data and parse it into map. + */ + private Map readAndParseStructuredData() throws IOException { + int c = read(true); + + if (c == '-') { + return null; + } + + ByteArrayOutputStream ret = new ByteArrayOutputStream(128); + + if (c != '[') { + throw new IOException("Unexpected syslog character: " + (char) c); + } + + Map structuredData = new HashMap<>(); + while (c == '[') { + + // Read SD-ID + while ((c = read(true)) != ' ' && c != ']') { + ret.write(c); + } + + String sdId = new String(ret.toByteArray()); + structuredData.put("sdId", sdId); + ret.reset(); + + while (c == ' ') { + // Read PARAM-NAME + while ((c = read(true)) != '=') { + ret.write(c); + } + + String sdKey = new String(ret.toByteArray()); + ret.reset(); + + expect('"'); + + // Read PARAM-DATA + while ((c = read(true)) != '"') { + ret.write(c); + + if (c == '\\') { + c = read(true); + ret.write(c); + } + } + + String sdValue = new String(ret.toByteArray()); + ret.reset(); + structuredData.put(sdKey, StringEscapeUtils.unescapeHtml4(sdValue)); + + c = read(true); + } + + if (c != ']') { + throw new IOException("Unexpected syslog character: " + (char) c); + } + + c = read(false); + } + + if (c != -1) { + unread(c); + } + + return structuredData; + } + + public void setInputStream(final InputStream in) { + this.in = in; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/syslog/SyslogSerDe.java b/ql/src/java/org/apache/hadoop/hive/ql/log/syslog/SyslogSerDe.java new file mode 100644 index 0000000000..8523c41b50 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/log/syslog/SyslogSerDe.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.log.syslog; + +import java.io.ByteArrayInputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; + +import javax.annotation.Nullable; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde2.AbstractSerDe; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.SerDeStats; +import org.apache.hadoop.hive.serde2.SerDeUtils; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; + + +public class SyslogSerDe extends AbstractSerDe { + private static final String COLUMN_NAMES = + "facility,severity,version,ts,hostname,app_name,proc_id,msg_id,structured_data,msg,unmatched"; + private static final String COLUMN_NAME_DELIMITER = String.valueOf(SerDeUtils.COMMA); + private static final String COLUMN_TYPES = + "string:string:string:timestamp:string:string:string:string:map:binary:binary"; + + private ObjectInspector inspector; + private SyslogParser syslogParser; + private List EMPTY_ROW; + + @Override + public void initialize(@Nullable final Configuration configuration, final Properties properties) + throws SerDeException { + + final List columnNames = Arrays.asList(COLUMN_NAMES.split(COLUMN_NAME_DELIMITER)); + final List columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(COLUMN_TYPES); + + EMPTY_ROW = new ArrayList<>(columnNames.size()); + for (int i = 0; i < columnNames.size(); i++) { + EMPTY_ROW.add(null); + } + StructTypeInfo typeInfo = (StructTypeInfo) TypeInfoFactory.getStructTypeInfo(columnNames, columnTypes); + this.inspector = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(typeInfo); + syslogParser = new SyslogParser(); + } + + @Override + public Class getSerializedClass() { + return Text.class; + } + + @Override + public Writable serialize(final Object o, final ObjectInspector objectInspector) throws SerDeException { + throw new SerDeException("Serialization is not supported yet"); + } + + @Override + public SerDeStats getSerDeStats() { + return null; + } + + @Override + public Object deserialize(final Writable writable) throws SerDeException { + Text rowText = (Text) writable; + if (rowText.getLength() == 0) { + // add the empty byte[] as unmatched line + EMPTY_ROW.set(EMPTY_ROW.size() - 1, rowText.getBytes()); + return EMPTY_ROW; + } + ByteArrayInputStream bis = new ByteArrayInputStream(rowText.getBytes(), 0, rowText.getLength()); + syslogParser.setInputStream(bis); + try { + return syslogParser.readEvent(); + } catch (Exception e) { + throw new SerDeException("Failed parsing line: " + rowText.toString(), e); + } + } + + @Override + public ObjectInspector getObjectInspector() { + return inspector; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/syslog/SyslogStorageHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/log/syslog/SyslogStorageHandler.java new file mode 100644 index 0000000000..a0e84d1dbd --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/log/syslog/SyslogStorageHandler.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.log.syslog; + +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.HiveMetaHook; +import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; +import org.apache.hadoop.hive.ql.plan.TableDesc; +import org.apache.hadoop.hive.ql.security.authorization.DefaultHiveAuthorizationProvider; +import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider; +import org.apache.hadoop.hive.serde2.AbstractSerDe; +import org.apache.hadoop.mapred.InputFormat; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.OutputFormat; +import org.apache.hadoop.mapred.TextOutputFormat; + +/** + * SyslogStorageHandler handles log files written to logs table in sys db. sys.logs is external table partitioned by + * dt (date), ns (namespace) and app (application) where logs files are stored in YYYY-mm-DD-HH-MM_i[.log][.gz] format. + * The contents in the log files are expected to be in RFC5424 layout for machine readability and + * human readability (some editors can replace the unescape the linebreaks correctly to show stacktraces etc.). + * NOTE: This storage handler is not generic and has several assumptions that are specific for hive's own log + * processing. Refer {@link SyslogInputFormat} and {@link SyslogParser} for more assumptions. + */ +public class SyslogStorageHandler implements HiveStorageHandler { + private Configuration conf; + + @Override + public Class getInputFormatClass() { + return SyslogInputFormat.class; + } + + @Override + public Class getOutputFormatClass() { + return TextOutputFormat.class; + } + + @Override + public Class getSerDeClass() { + return SyslogSerDe.class; + } + + @Override + public HiveMetaHook getMetaHook() { + // no hook by default + return null; + } + + public HiveAuthorizationProvider getAuthorizationProvider() { + return new DefaultHiveAuthorizationProvider(); + } + + @Override + public void configureInputJobProperties(TableDesc tableDesc, + Map jobProperties) { + // do nothing by default + } + + @Override + public void configureOutputJobProperties(TableDesc tableDesc, + Map jobProperties) { + // do nothing by default + } + + @Override + public void configureTableJobProperties(TableDesc tableDesc, + Map jobProperties) { + //do nothing by default + } + + @Override + public void configureJobConf(TableDesc tableDesc, JobConf jobConf) { + //do nothing by default + } + + @Override + public void configureInputJobCredentials(TableDesc tableDesc, Map secrets) { + //do nothing by default + } + + @Override + public Configuration getConf() { + return conf; + } + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + } + + @Override + public String toString() { + return this.getClass().getName(); + } +} diff --git a/ql/src/test/org/apache/hadoop/hive/ql/log/TestSyslogInputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/log/TestSyslogInputFormat.java new file mode 100644 index 0000000000..01ffba368b --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/log/TestSyslogInputFormat.java @@ -0,0 +1,391 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.log; + +import java.sql.Timestamp; +import java.util.List; +import java.util.TimeZone; +import java.util.stream.Collectors; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; +import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; +import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory; +import org.apache.hadoop.hive.ql.log.syslog.SyslogInputFormat; +import org.junit.Assert; +import org.junit.Test; + +import com.google.common.collect.Lists; + +public class TestSyslogInputFormat { + + @Test + public void testTimestampFilePruningAllBefore() { + TimeZone defaultTZ = TimeZone.getDefault(); + try { + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + SearchArgument sarg = SearchArgumentFactory.newBuilder().between("ts", PredicateLeaf.Type.TIMESTAMP, + Timestamp.valueOf("2018-01-03 10:00:00.000"), Timestamp.valueOf("2019-01-03 10:00:00.000")).build(); + SyslogInputFormat syslogInputFormat = new SyslogInputFormat(); + FileStatus fs1 = new FileStatus(); + fs1.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-10_0.log.gz")); + FileStatus fs2 = new FileStatus(); + fs2.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-15_0.log.gz")); + FileStatus fs3 = new FileStatus(); + fs3.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-15_1.log.gz")); + FileStatus fs4 = new FileStatus(); + fs4.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-35_0.log.gz")); + FileStatus fs5 = new FileStatus(); + fs5.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-11-15_0.log.gz")); + FileStatus fs6 = new FileStatus(); + fs6.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-11-35_0.log.gz")); + List inputFiles = Lists.newArrayList(fs1, fs2, fs3, fs4, fs5, fs6); + List expectedFiles = Lists.newArrayList(); + List selectedFiles = syslogInputFormat.pruneFiles(sarg, 300, inputFiles); + List gotFiles = selectedFiles + .stream() + .map(f -> f.getPath().toString()) + .collect(Collectors.toList()); + Assert.assertEquals(expectedFiles, gotFiles); + } finally { + TimeZone.setDefault(defaultTZ); + } + } + + @Test + public void testTimestampFilePruningAllAfter() { + TimeZone defaultTZ = TimeZone.getDefault(); + try { + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + SearchArgument sarg = SearchArgumentFactory.newBuilder().between("ts", PredicateLeaf.Type.TIMESTAMP, + Timestamp.valueOf("2020-01-03 10:00:00.000"), Timestamp.valueOf("2020-01-03 10:00:00.000")).build(); + SyslogInputFormat syslogInputFormat = new SyslogInputFormat(); + FileStatus fs1 = new FileStatus(); + fs1.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-10_0.log.gz")); + FileStatus fs2 = new FileStatus(); + fs2.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-15_0.log.gz")); + FileStatus fs3 = new FileStatus(); + fs3.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-15_1.log.gz")); + FileStatus fs4 = new FileStatus(); + fs4.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-35_0.log.gz")); + FileStatus fs5 = new FileStatus(); + fs5.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-11-15_0.log.gz")); + FileStatus fs6 = new FileStatus(); + fs6.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-11-35_0.log.gz")); + List inputFiles = Lists.newArrayList(fs1, fs2, fs3, fs4, fs5, fs6); + List expectedFiles = Lists.newArrayList(); + List selectedFiles = syslogInputFormat.pruneFiles(sarg, 300, inputFiles); + List gotFiles = selectedFiles + .stream() + .map(f -> f.getPath().toString()) + .collect(Collectors.toList()); + Assert.assertEquals(expectedFiles, gotFiles); + } finally { + TimeZone.setDefault(defaultTZ); + } + } + + @Test + public void testTimestampFilePruningSimple() { + TimeZone defaultTZ = TimeZone.getDefault(); + try { + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + SearchArgument sarg = SearchArgumentFactory.newBuilder().between("ts", PredicateLeaf.Type.TIMESTAMP, + Timestamp.valueOf("2019-01-03 10:15:00.000"), Timestamp.valueOf("2019-01-03 11:15:00.000")).build(); + SyslogInputFormat syslogInputFormat = new SyslogInputFormat(); + FileStatus fs1 = new FileStatus(); + fs1.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-00_0.log.gz")); + FileStatus fs2 = new FileStatus(); + fs2.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-15_0.log.gz")); + FileStatus fs3 = new FileStatus(); + fs3.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-15_1.log.gz")); + FileStatus fs4 = new FileStatus(); + fs4.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-35_0.log.gz")); + FileStatus fs5 = new FileStatus(); + fs5.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-11-15_0.log.gz")); + FileStatus fs6 = new FileStatus(); + fs6.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-11-35_0.log.gz")); + List inputFiles = Lists.newArrayList(fs1, fs2, fs3, fs4, fs5, fs6); + List expectedFiles = Lists.newArrayList(fs2, fs3, fs4, fs5) + .stream() + .map(f -> f.getPath().toString()) + .collect(Collectors.toList()); + List selectedFiles = syslogInputFormat.pruneFiles(sarg, 300, inputFiles); + List gotFiles = selectedFiles + .stream() + .map(f -> f.getPath().toString()) + .collect(Collectors.toList()); + Assert.assertEquals(expectedFiles, gotFiles); + } finally { + TimeZone.setDefault(defaultTZ); + } + } + + @Test + public void testTimestampFilePruningTimeSlice() { + TimeZone defaultTZ = TimeZone.getDefault(); + try { + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + SearchArgument sarg = SearchArgumentFactory.newBuilder().between("ts", PredicateLeaf.Type.TIMESTAMP, + Timestamp.valueOf("2019-01-03 10:16:00.000"), Timestamp.valueOf("2019-01-03 11:16:00.000")).build(); + SyslogInputFormat syslogInputFormat = new SyslogInputFormat(); + FileStatus fs1 = new FileStatus(); + fs1.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-10_0.log.gz")); + FileStatus fs2 = new FileStatus(); + fs2.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-15_0.log.gz")); + FileStatus fs3 = new FileStatus(); + fs3.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-15_1.log.gz")); + FileStatus fs4 = new FileStatus(); + fs4.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-35_0.log.gz")); + FileStatus fs5 = new FileStatus(); + fs5.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-11-15_0.log.gz")); + FileStatus fs6 = new FileStatus(); + fs6.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-11-35_0.log.gz")); + List inputFiles = Lists.newArrayList(fs1, fs2, fs3, fs4, fs5, fs6); + List expectedFiles = Lists.newArrayList(fs2, fs3, fs4, fs5) + .stream() + .map(f -> f.getPath().toString()) + .collect(Collectors.toList()); + List selectedFiles = syslogInputFormat.pruneFiles(sarg, 300, inputFiles); + List gotFiles = selectedFiles + .stream() + .map(f -> f.getPath().toString()) + .collect(Collectors.toList()); + Assert.assertEquals(expectedFiles, gotFiles); + } finally { + TimeZone.setDefault(defaultTZ); + } + } + + @Test + public void testTimestampFilePruningPredicateWithSeconds() { + TimeZone defaultTZ = TimeZone.getDefault(); + try { + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + SearchArgument sarg = SearchArgumentFactory.newBuilder().between("ts", PredicateLeaf.Type.TIMESTAMP, + Timestamp.valueOf("2019-01-03 10:15:03.600"), Timestamp.valueOf("2019-01-03 11:15:04.320")).build(); + SyslogInputFormat syslogInputFormat = new SyslogInputFormat(); + FileStatus fs1 = new FileStatus(); + fs1.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-00_0.log.gz")); + FileStatus fs2 = new FileStatus(); + fs2.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-15_0.log.gz")); + FileStatus fs3 = new FileStatus(); + fs3.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-15_1.log.gz")); + FileStatus fs4 = new FileStatus(); + fs4.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-35_0.log.gz")); + FileStatus fs5 = new FileStatus(); + fs5.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-11-15_0.log.gz")); + FileStatus fs6 = new FileStatus(); + fs6.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-11-35_0.log.gz")); + List inputFiles = Lists.newArrayList(fs1, fs2, fs3, fs4, fs5, fs6); + List expectedFiles = Lists.newArrayList(fs2, fs3, fs4, fs5) + .stream() + .map(f -> f.getPath().toString()) + .collect(Collectors.toList()); + List selectedFiles = syslogInputFormat.pruneFiles(sarg, 300, inputFiles); + List gotFiles = selectedFiles + .stream() + .map(f -> f.getPath().toString()) + .collect(Collectors.toList()); + Assert.assertEquals(expectedFiles, gotFiles); + } finally { + TimeZone.setDefault(defaultTZ); + } + } + + @Test + public void testTimestampFilePruningPredicateWithSecondsFlipRange() { + TimeZone defaultTZ = TimeZone.getDefault(); + try { + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + SearchArgument sarg = SearchArgumentFactory.newBuilder().between("ts", PredicateLeaf.Type.TIMESTAMP, + Timestamp.valueOf("2019-01-03 11:15:04.320"), Timestamp.valueOf("2019-01-03 10:15:03.600")).build(); + SyslogInputFormat syslogInputFormat = new SyslogInputFormat(); + FileStatus fs1 = new FileStatus(); + fs1.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-00_0.log.gz")); + FileStatus fs2 = new FileStatus(); + fs2.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-15_0.log.gz")); + FileStatus fs3 = new FileStatus(); + fs3.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-15_1.log.gz")); + FileStatus fs4 = new FileStatus(); + fs4.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-35_0.log.gz")); + FileStatus fs5 = new FileStatus(); + fs5.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-11-15_0.log.gz")); + FileStatus fs6 = new FileStatus(); + fs6.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-11-35_0.log.gz")); + List inputFiles = Lists.newArrayList(fs1, fs2, fs3, fs4, fs5, fs6); + List expectedFiles = Lists.newArrayList(); + List selectedFiles = syslogInputFormat.pruneFiles(sarg, 300, inputFiles); + List gotFiles = selectedFiles + .stream() + .map(f -> f.getPath().toString()) + .collect(Collectors.toList()); + Assert.assertEquals(expectedFiles, gotFiles); + } finally { + TimeZone.setDefault(defaultTZ); + } + } + + @Test + public void testTimestampNoFilePruningUnsupportedColumnName() { + TimeZone defaultTZ = TimeZone.getDefault(); + try { + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + SearchArgument sarg = SearchArgumentFactory.newBuilder().between("ts1", PredicateLeaf.Type.TIMESTAMP, + Timestamp.valueOf("2019-01-03 11:15:04.320"), Timestamp.valueOf("2019-01-03 10:15:03.600")).build(); + SyslogInputFormat syslogInputFormat = new SyslogInputFormat(); + FileStatus fs1 = new FileStatus(); + fs1.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-10_0.log.gz")); + FileStatus fs2 = new FileStatus(); + fs2.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-15_0.log.gz")); + FileStatus fs3 = new FileStatus(); + fs3.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-15_1.log.gz")); + FileStatus fs4 = new FileStatus(); + fs4.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-35_0.log.gz")); + FileStatus fs5 = new FileStatus(); + fs5.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-11-15_0.log.gz")); + FileStatus fs6 = new FileStatus(); + fs6.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-11-35_0.log.gz")); + List inputFiles = Lists.newArrayList(fs1, fs2, fs3, fs4, fs5, fs6); + List expectedFiles = Lists.newArrayList(fs1, fs2, fs3, fs4, fs5, fs6) + .stream() + .map(f -> f.getPath().toString()) + .collect(Collectors.toList()); + List selectedFiles = syslogInputFormat.pruneFiles(sarg, 300, inputFiles); + List gotFiles = selectedFiles + .stream() + .map(f -> f.getPath().toString()) + .collect(Collectors.toList()); + Assert.assertEquals(expectedFiles, gotFiles); + } finally { + TimeZone.setDefault(defaultTZ); + } + } + + @Test + public void testTimestampNoFilePruningUnsupportedExpression() { + TimeZone defaultTZ = TimeZone.getDefault(); + try { + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + SearchArgument sarg = SearchArgumentFactory.newBuilder().equals("ts", PredicateLeaf.Type.TIMESTAMP, + Timestamp.valueOf("2019-01-03 11:15:04.320")).build(); + SyslogInputFormat syslogInputFormat = new SyslogInputFormat(); + FileStatus fs1 = new FileStatus(); + fs1.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-10_0.log.gz")); + FileStatus fs2 = new FileStatus(); + fs2.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-15_0.log.gz")); + FileStatus fs3 = new FileStatus(); + fs3.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-15_1.log.gz")); + FileStatus fs4 = new FileStatus(); + fs4.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-35_0.log.gz")); + FileStatus fs5 = new FileStatus(); + fs5.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-11-15_0.log.gz")); + FileStatus fs6 = new FileStatus(); + fs6.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-11-35_0.log.gz")); + List inputFiles = Lists.newArrayList(fs1, fs2, fs3, fs4, fs5, fs6); + List expectedFiles = Lists.newArrayList(fs1, fs2, fs3, fs4, fs5, fs6) + .stream() + .map(f -> f.getPath().toString()) + .collect(Collectors.toList()); + List selectedFiles = syslogInputFormat.pruneFiles(sarg, 300, inputFiles); + List gotFiles = selectedFiles + .stream() + .map(f -> f.getPath().toString()) + .collect(Collectors.toList()); + Assert.assertEquals(expectedFiles, gotFiles); + } finally { + TimeZone.setDefault(defaultTZ); + } + } + + @Test + public void testTimestampNoFilePruningUnsupportedMultiExpression() { + TimeZone defaultTZ = TimeZone.getDefault(); + try { + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + SearchArgument sarg = SearchArgumentFactory.newBuilder().startOr() + .equals("ts", PredicateLeaf.Type.TIMESTAMP, Timestamp.valueOf("2019-01-03 11:15:04.320")) + .equals("ts", PredicateLeaf.Type.TIMESTAMP, Timestamp.valueOf("2020-01-03 11:15:04.320")) + .end() + .build(); + SyslogInputFormat syslogInputFormat = new SyslogInputFormat(); + FileStatus fs1 = new FileStatus(); + fs1.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-10_0.log.gz")); + FileStatus fs2 = new FileStatus(); + fs2.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-15_0.log.gz")); + FileStatus fs3 = new FileStatus(); + fs3.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-15_1.log.gz")); + FileStatus fs4 = new FileStatus(); + fs4.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-35_0.log.gz")); + FileStatus fs5 = new FileStatus(); + fs5.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-11-15_0.log.gz")); + FileStatus fs6 = new FileStatus(); + fs6.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-11-35_0.log.gz")); + List inputFiles = Lists.newArrayList(fs1, fs2, fs3, fs4, fs5, fs6); + List expectedFiles = Lists.newArrayList(fs1, fs2, fs3, fs4, fs5, fs6) + .stream() + .map(f -> f.getPath().toString()) + .collect(Collectors.toList()); + List selectedFiles = syslogInputFormat.pruneFiles(sarg, 300, inputFiles); + List gotFiles = selectedFiles + .stream() + .map(f -> f.getPath().toString()) + .collect(Collectors.toList()); + Assert.assertEquals(expectedFiles, gotFiles); + } finally { + TimeZone.setDefault(defaultTZ); + } + } + + @Test + public void testTimestampNoFilePruningUnsupportedFilename() { + TimeZone defaultTZ = TimeZone.getDefault(); + try { + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + SearchArgument sarg = SearchArgumentFactory.newBuilder().between("ts", PredicateLeaf.Type.TIMESTAMP, + Timestamp.valueOf("2019-01-03 10:15:03.600"), Timestamp.valueOf("2019-01-03 11:15:04.320")).build(); + SyslogInputFormat syslogInputFormat = new SyslogInputFormat(); + FileStatus fs1 = new FileStatus(); + fs1.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-10")); + FileStatus fs2 = new FileStatus(); + fs2.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-15")); + FileStatus fs3 = new FileStatus(); + fs3.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-15")); + FileStatus fs4 = new FileStatus(); + fs4.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-10-35")); + FileStatus fs5 = new FileStatus(); + fs5.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-11-15")); + FileStatus fs6 = new FileStatus(); + fs6.setPath(new Path("s3a://bucket/logs/dt=2019-01-01/ns=foo/app=hs2/2019-01-03-11-35")); + List inputFiles = Lists.newArrayList(fs1, fs2, fs3, fs4, fs5, fs6); + List expectedFiles = Lists.newArrayList(fs1, fs2, fs3, fs4, fs5, fs6) + .stream() + .map(f -> f.getPath().toString()) + .collect(Collectors.toList()); + List selectedFiles = syslogInputFormat.pruneFiles(sarg, 300, inputFiles); + List gotFiles = selectedFiles + .stream() + .map(f -> f.getPath().toString()) + .collect(Collectors.toList()); + Assert.assertEquals(expectedFiles, gotFiles); + } finally { + TimeZone.setDefault(defaultTZ); + } + } +} diff --git a/ql/src/test/queries/clientpositive/syslog_parser.q b/ql/src/test/queries/clientpositive/syslog_parser.q new file mode 100644 index 0000000000..78d3e0d1c3 --- /dev/null +++ b/ql/src/test/queries/clientpositive/syslog_parser.q @@ -0,0 +1,81 @@ +set hive.fetch.task.conversion=none; + +-- SORT_QUERY_RESULTS + +CREATE EXTERNAL TABLE logs( +facility STRING, +severity STRING, +version STRING, +ts TIMESTAMP, +hostname STRING, +app_name STRING, +proc_id STRING, +msg_id STRING, +structured_data MAP, +msg BINARY, +unmatched BINARY +) +ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.log.syslog.SyslogSerDe' +STORED AS TEXTFILE; + +load data local inpath '../../data/files/syslog-hs2.log' into table logs; +load data local inpath '../../data/files/syslog-hs2-2.log' into table logs; + +select count(*) from logs where severity="ERROR"; +select count(*) from logs; +select count(*) from logs where unmatched is null; +select count(*) from logs where unmatched is not null; +select severity,count(*) from logs group by severity order by severity; +select severity,count(*) from logs where ts between "2019-03-21 07:00:00" and "2019-03-21 07:06:00" group by severity; +select severity,count(*) from logs where ts between "2019-03-21 07:06:00" and "2019-03-21 07:07:00" group by severity; +select severity,count(*) from logs where ts between "2019-03-21 07:07:00" and "2019-03-21 07:08:00" group by severity; +select severity,count(*) from logs where ts between "2019-03-21 07:08:00" and "2019-03-21 08:08:00" group by severity; +select severity,count(*) from logs where ts between "2019-03-22 01:00:00" and "2019-03-21 01:08:00" group by severity; +select severity,count(*) from logs where ts between "2019-03-22 01:08:00" and "2019-03-21 01:09:00" group by severity; +select severity,count(*) from logs where ts between "2019-03-22 01:09:00" and "2019-03-21 01:10:00" group by severity; +select ts,severity,structured_data["thread"],structured_data["class"] from logs where severity="ERROR" ORDER BY ts; +select ts,severity,structured_data["thread"],structured_data["class"],app_name from logs where severity="WARN" ORDER BY ts; +select decode(unmatched, 'UTF-8') from logs where unmatched is not null limit 10; + +CREATE EXTERNAL TABLE logs2( +facility STRING, +severity STRING, +version STRING, +ts TIMESTAMP, +hostname STRING, +app_name STRING, +proc_id STRING, +msg_id STRING, +structured_data MAP, +msg BINARY, +unmatched BINARY +) +PARTITIONED BY(dt DATE,ns STRING,app STRING) +STORED BY 'org.apache.hadoop.hive.ql.log.syslog.SyslogStorageHandler' +LOCATION "${hiveconf:hive.metastore.warehouse.dir}/logs2"; + +dfs -mkdir -p ${hiveconf:hive.metastore.warehouse.dir}/logs2/dt=2019-01-03/ns=foo/app=hs2/; +dfs -cp '../../data/files/syslog-hs2.log' ${hiveconf:hive.metastore.warehouse.dir}/logs2/dt=2019-01-03/ns=foo/app=hs2/2019-03-21-07-05.log; +dfs -mkdir -p ${hiveconf:hive.metastore.warehouse.dir}/logs2/dt=2019-11-23/ns=bar/app=hs2/; +dfs -cp '../../data/files/syslog-hs2.log' ${hiveconf:hive.metastore.warehouse.dir}/logs2/dt=2019-11-23/ns=bar/app=hs2/2019-02-22-01-05.log; + +MSCK REPAIR TABLE logs2; + +select severity,count(*) from logs2 group by severity order by severity; +select count(*) from logs2 where severity="INFO" and dt='2019-01-03'; +select count(*) from logs2 where severity="INFO" and ns='foo'; +select count(*) from logs2 where severity="INFO" and dt='2019-11-23'; +select count(*) from logs2 where severity="INFO" and ns='bar'; +select count(*) from logs2 where severity="INFO"; +select count(*) from logs2 where severity="INFO" and app='hs2'; +select count(*) from logs2 where severity="INFO" and app='non-existent'; +select count(*) from logs2 where ns='foo'; +select count(*) from logs2 where ns='bar'; +select count(*) from logs2 where unmatched is null and dt between '2019-01-03' and '2019-11-23'; +select count(*) from logs2 where unmatched is not null; +select ts,severity,structured_data["thread"],structured_data["class"] from logs2 where severity="ERROR" order by ts; +select ts,severity,structured_data["thread"],structured_data["class"],app_name from logs2 where severity="WARN" order by ts; +select decode(unmatched, 'UTF-8') from logs where unmatched is not null limit 10; + +drop table logs; +drop table logs2; diff --git a/ql/src/test/queries/clientpositive/syslog_parser_file_pruning.q b/ql/src/test/queries/clientpositive/syslog_parser_file_pruning.q new file mode 100644 index 0000000000..11bcbb7635 --- /dev/null +++ b/ql/src/test/queries/clientpositive/syslog_parser_file_pruning.q @@ -0,0 +1,82 @@ +set hive.fetch.task.conversion=none; + +set time zone UTC; + +-- SORT_QUERY_RESULTS + +CREATE EXTERNAL TABLE logs2( +facility STRING, +severity STRING, +version STRING, +ts TIMESTAMP, +hostname STRING, +app_name STRING, +proc_id STRING, +msg_id STRING, +structured_data MAP, +msg BINARY, +unmatched BINARY +) +PARTITIONED BY(dt DATE,ns STRING,app STRING) +STORED BY 'org.apache.hadoop.hive.ql.log.syslog.SyslogStorageHandler' +LOCATION "${hiveconf:hive.metastore.warehouse.dir}/logs2"; + +dfs -mkdir -p ${hiveconf:hive.metastore.warehouse.dir}/logs2/dt=2019-03-21/ns=foo/app=hs2/; +dfs -cp '../../data/files/syslog-hs2.log' ${hiveconf:hive.metastore.warehouse.dir}/logs2/dt=2019-03-21/ns=foo/app=hs2/2019-03-21-07-05_0.log; +dfs -mkdir -p ${hiveconf:hive.metastore.warehouse.dir}/logs2/dt=2019-03-22/ns=bar/app=hs2/; +dfs -cp '../../data/files/syslog-hs2-2.log' ${hiveconf:hive.metastore.warehouse.dir}/logs2/dt=2019-03-22/ns=bar/app=hs2/2019-03-22-01-05_0.log; + +MSCK REPAIR TABLE logs2; + +SET hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecTezSummaryPrinter; +set hive.syslog.input.format.file.pruning=false; +-- before +select severity,count(*) from logs2 where ts between '2019-03-21 07:00:00.0' and '2019-03-21 07:06:00.0' group by severity; +-- middle +select severity,count(*) from logs2 where ts between '2019-03-21 07:06:00.0' and '2019-03-21 07:07:00.0' group by severity; +-- middle +select severity,count(*) from logs2 where ts between '2019-03-21 07:07:00.0' and '2019-03-21 07:08:00.0' group by severity; +-- after +select severity,count(*) from logs2 where ts between '2019-03-21 07:08:00.0' and '2019-03-21 08:08:00.0' group by severity; +-- all +select severity,count(*) from logs2 where ts between '2019-03-21 07:00:00.0' and '2019-03-21 08:00:00.0' group by severity; +-- all no filter +select severity,count(*) from logs2 where dt='2019-03-21' group by severity; + +-- before +select severity,count(*) from logs2 where ts between '2019-03-22 01:00:00.0' and '2019-03-22 01:08:00.0' group by severity; +-- middle +select severity,count(*) from logs2 where ts between '2019-03-22 01:08:00.0' and '2019-03-22 01:09:00.0' group by severity; +-- after +select severity,count(*) from logs2 where ts between '2019-03-22 01:09:00.0' and '2019-03-22 01:10:00.0' group by severity; +-- all +select severity,count(*) from logs2 where ts between '2019-03-22 01:00:00.0' and '2019-03-22 02:00:00.0' group by severity; +-- all no filter +select severity,count(*) from logs2 where dt='2019-03-22' group by severity; + +set hive.syslog.input.format.file.pruning=true; +-- before +select severity,count(*) from logs2 where ts between '2019-03-21 07:00:00.0' and '2019-03-21 07:06:00.0' group by severity; +-- middle +select severity,count(*) from logs2 where ts between '2019-03-21 07:06:00.0' and '2019-03-21 07:07:00.0' group by severity; +-- middle +select severity,count(*) from logs2 where ts between '2019-03-21 07:07:00.0' and '2019-03-21 07:08:00.0' group by severity; +-- after +select severity,count(*) from logs2 where ts between '2019-03-21 07:08:00.0' and '2019-03-21 08:08:00.0' group by severity; +-- all +select severity,count(*) from logs2 where ts between '2019-03-21 07:00:00.0' and '2019-03-21 08:00:00.0' group by severity; +-- all no filter +select severity,count(*) from logs2 where dt='2019-03-21' group by severity; + +-- before +select severity,count(*) from logs2 where ts between '2019-03-22 01:00:00.0' and '2019-03-22 01:08:00.0' group by severity; +-- middle +select severity,count(*) from logs2 where ts between '2019-03-22 01:08:00.0' and '2019-03-22 01:09:00.0' group by severity; +-- after +select severity,count(*) from logs2 where ts between '2019-03-22 01:09:00.0' and '2019-03-22 01:10:00.0' group by severity; +-- all +select severity,count(*) from logs2 where ts between '2019-03-22 01:00:00.0' and '2019-03-22 02:00:00.0' group by severity; +-- all no filter +select severity,count(*) from logs2 where dt='2019-03-22' group by severity; + +drop table logs2; diff --git a/ql/src/test/results/clientpositive/llap/syslog_parser.q.out b/ql/src/test/results/clientpositive/llap/syslog_parser.q.out new file mode 100644 index 0000000000..420a0ca033 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/syslog_parser.q.out @@ -0,0 +1,515 @@ +PREHOOK: query: CREATE EXTERNAL TABLE logs( +facility STRING, +severity STRING, +version STRING, +ts TIMESTAMP, +hostname STRING, +app_name STRING, +proc_id STRING, +msg_id STRING, +structured_data MAP, +msg BINARY, +unmatched BINARY +) +ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.log.syslog.SyslogSerDe' +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@logs +POSTHOOK: query: CREATE EXTERNAL TABLE logs( +facility STRING, +severity STRING, +version STRING, +ts TIMESTAMP, +hostname STRING, +app_name STRING, +proc_id STRING, +msg_id STRING, +structured_data MAP, +msg BINARY, +unmatched BINARY +) +ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.log.syslog.SyslogSerDe' +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@logs +PREHOOK: query: load data local inpath '../../data/files/syslog-hs2.log' into table logs +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@logs +POSTHOOK: query: load data local inpath '../../data/files/syslog-hs2.log' into table logs +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@logs +PREHOOK: query: load data local inpath '../../data/files/syslog-hs2-2.log' into table logs +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@logs +POSTHOOK: query: load data local inpath '../../data/files/syslog-hs2-2.log' into table logs +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@logs +PREHOOK: query: select count(*) from logs where severity="ERROR" +PREHOOK: type: QUERY +PREHOOK: Input: default@logs +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from logs where severity="ERROR" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs +#### A masked pattern was here #### +3 +PREHOOK: query: select count(*) from logs +PREHOOK: type: QUERY +PREHOOK: Input: default@logs +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from logs +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs +#### A masked pattern was here #### +1137 +PREHOOK: query: select count(*) from logs where unmatched is null +PREHOOK: type: QUERY +PREHOOK: Input: default@logs +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from logs where unmatched is null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs +#### A masked pattern was here #### +939 +PREHOOK: query: select count(*) from logs where unmatched is not null +PREHOOK: type: QUERY +PREHOOK: Input: default@logs +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from logs where unmatched is not null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs +#### A masked pattern was here #### +198 +PREHOOK: query: select severity,count(*) from logs group by severity order by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs +#### A masked pattern was here #### +POSTHOOK: query: select severity,count(*) from logs group by severity order by severity +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs +#### A masked pattern was here #### +DEBUG 188 +ERROR 3 +INFO 726 +NULL 198 +WARN 22 +PREHOOK: query: select severity,count(*) from logs where ts between "2019-03-21 07:00:00" and "2019-03-21 07:06:00" group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs +#### A masked pattern was here #### +POSTHOOK: query: select severity,count(*) from logs where ts between "2019-03-21 07:00:00" and "2019-03-21 07:06:00" group by severity +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs +#### A masked pattern was here #### +PREHOOK: query: select severity,count(*) from logs where ts between "2019-03-21 07:06:00" and "2019-03-21 07:07:00" group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs +#### A masked pattern was here #### +POSTHOOK: query: select severity,count(*) from logs where ts between "2019-03-21 07:06:00" and "2019-03-21 07:07:00" group by severity +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs +#### A masked pattern was here #### +DEBUG 50 +INFO 376 +WARN 19 +PREHOOK: query: select severity,count(*) from logs where ts between "2019-03-21 07:07:00" and "2019-03-21 07:08:00" group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs +#### A masked pattern was here #### +POSTHOOK: query: select severity,count(*) from logs where ts between "2019-03-21 07:07:00" and "2019-03-21 07:08:00" group by severity +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs +#### A masked pattern was here #### +DEBUG 107 +ERROR 3 +INFO 161 +WARN 3 +PREHOOK: query: select severity,count(*) from logs where ts between "2019-03-21 07:08:00" and "2019-03-21 08:08:00" group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs +#### A masked pattern was here #### +POSTHOOK: query: select severity,count(*) from logs where ts between "2019-03-21 07:08:00" and "2019-03-21 08:08:00" group by severity +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs +#### A masked pattern was here #### +PREHOOK: query: select severity,count(*) from logs where ts between "2019-03-22 01:00:00" and "2019-03-21 01:08:00" group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs +#### A masked pattern was here #### +POSTHOOK: query: select severity,count(*) from logs where ts between "2019-03-22 01:00:00" and "2019-03-21 01:08:00" group by severity +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs +#### A masked pattern was here #### +PREHOOK: query: select severity,count(*) from logs where ts between "2019-03-22 01:08:00" and "2019-03-21 01:09:00" group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs +#### A masked pattern was here #### +POSTHOOK: query: select severity,count(*) from logs where ts between "2019-03-22 01:08:00" and "2019-03-21 01:09:00" group by severity +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs +#### A masked pattern was here #### +PREHOOK: query: select severity,count(*) from logs where ts between "2019-03-22 01:09:00" and "2019-03-21 01:10:00" group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs +#### A masked pattern was here #### +POSTHOOK: query: select severity,count(*) from logs where ts between "2019-03-22 01:09:00" and "2019-03-21 01:10:00" group by severity +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs +#### A masked pattern was here #### +PREHOOK: query: select ts,severity,structured_data["thread"],structured_data["class"] from logs where severity="ERROR" ORDER BY ts +PREHOOK: type: QUERY +PREHOOK: Input: default@logs +#### A masked pattern was here #### +POSTHOOK: query: select ts,severity,structured_data["thread"],structured_data["class"] from logs where severity="ERROR" ORDER BY ts +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs +#### A masked pattern was here #### +2019-03-21 07:07:09.857 ERROR HiveServer2-Background-Pool: Thread-85 metadata.Hive +2019-03-21 07:07:17.035 ERROR HiveServer2-Background-Pool: Thread-92 ql.Driver +2019-03-21 07:07:17.105 ERROR HiveServer2-Background-Pool: Thread-92 operation.Operation +PREHOOK: query: select ts,severity,structured_data["thread"],structured_data["class"],app_name from logs where severity="WARN" ORDER BY ts +PREHOOK: type: QUERY +PREHOOK: Input: default@logs +#### A masked pattern was here #### +POSTHOOK: query: select ts,severity,structured_data["thread"],structured_data["class"],app_name from logs where severity="WARN" ORDER BY ts +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs +#### A masked pattern was here #### +2019-03-21 07:06:15.136 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:15.137 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:15.137 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:15.284 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:15.284 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:15.284 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:18.555 WARN main session.SessionState hiveserver2 +2019-03-21 07:06:19.798 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:19.798 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:19.798 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:20.142 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:20.143 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:20.143 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:20.143 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:20.405 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:20.405 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:20.405 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:20.408 WARN main thrift.ThriftCLIService hiveserver2 +2019-03-21 07:06:20.626 WARN main utils.ZKPaths hiveserver2 +2019-03-21 07:07:20.145 WARN NotificationEventPoll 0 conf.HiveConf hiveserver2 +2019-03-21 07:07:20.145 WARN NotificationEventPoll 0 conf.HiveConf hiveserver2 +2019-03-21 07:07:20.146 WARN NotificationEventPoll 0 conf.HiveConf hiveserver2 +PREHOOK: query: select decode(unmatched, 'UTF-8') from logs where unmatched is not null limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@logs +#### A masked pattern was here #### +POSTHOOK: query: select decode(unmatched, 'UTF-8') from logs where unmatched is not null limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs +#### A masked pattern was here #### ++ '[' hiveserver2 == data-load ']'ator ']' ++ '[' hiveserver2 == query-coordinator ']' ++ '[' hiveserver2 == query-executor ']' ++ . /fluentd-utils.sh +#### A masked pattern was here #### +++ set -xntd-utils.sh +PREHOOK: query: CREATE EXTERNAL TABLE logs2( +facility STRING, +severity STRING, +version STRING, +ts TIMESTAMP, +hostname STRING, +app_name STRING, +proc_id STRING, +msg_id STRING, +structured_data MAP, +msg BINARY, +unmatched BINARY +) +PARTITIONED BY(dt DATE,ns STRING,app STRING) +STORED BY 'org.apache.hadoop.hive.ql.log.syslog.SyslogStorageHandler' +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@logs2 +POSTHOOK: query: CREATE EXTERNAL TABLE logs2( +facility STRING, +severity STRING, +version STRING, +ts TIMESTAMP, +hostname STRING, +app_name STRING, +proc_id STRING, +msg_id STRING, +structured_data MAP, +msg BINARY, +unmatched BINARY +) +PARTITIONED BY(dt DATE,ns STRING,app STRING) +STORED BY 'org.apache.hadoop.hive.ql.log.syslog.SyslogStorageHandler' +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@logs2 +PREHOOK: query: MSCK REPAIR TABLE logs2 +PREHOOK: type: MSCK +PREHOOK: Output: default@logs2 +POSTHOOK: query: MSCK REPAIR TABLE logs2 +POSTHOOK: type: MSCK +POSTHOOK: Output: default@logs2 +Partitions not in metastore: logs2:dt=2019-01-03/ns=foo/app=hs2 logs2:dt=2019-11-23/ns=bar/app=hs2 +#### A masked pattern was here #### +PREHOOK: query: select severity,count(*) from logs2 group by severity order by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-01-03/ns=foo/app=hs2 +PREHOOK: Input: default@logs2@dt=2019-11-23/ns=bar/app=hs2 +#### A masked pattern was here #### +POSTHOOK: query: select severity,count(*) from logs2 group by severity order by severity +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs2 +POSTHOOK: Input: default@logs2@dt=2019-01-03/ns=foo/app=hs2 +POSTHOOK: Input: default@logs2@dt=2019-11-23/ns=bar/app=hs2 +#### A masked pattern was here #### +DEBUG 314 +ERROR 6 +INFO 1074 +NULL 174 +WARN 44 +PREHOOK: query: select count(*) from logs2 where severity="INFO" and dt='2019-01-03' +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-01-03/ns=foo/app=hs2 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from logs2 where severity="INFO" and dt='2019-01-03' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs2 +POSTHOOK: Input: default@logs2@dt=2019-01-03/ns=foo/app=hs2 +#### A masked pattern was here #### +537 +PREHOOK: query: select count(*) from logs2 where severity="INFO" and ns='foo' +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-01-03/ns=foo/app=hs2 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from logs2 where severity="INFO" and ns='foo' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs2 +POSTHOOK: Input: default@logs2@dt=2019-01-03/ns=foo/app=hs2 +#### A masked pattern was here #### +537 +PREHOOK: query: select count(*) from logs2 where severity="INFO" and dt='2019-11-23' +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-11-23/ns=bar/app=hs2 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from logs2 where severity="INFO" and dt='2019-11-23' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs2 +POSTHOOK: Input: default@logs2@dt=2019-11-23/ns=bar/app=hs2 +#### A masked pattern was here #### +537 +PREHOOK: query: select count(*) from logs2 where severity="INFO" and ns='bar' +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-11-23/ns=bar/app=hs2 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from logs2 where severity="INFO" and ns='bar' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs2 +POSTHOOK: Input: default@logs2@dt=2019-11-23/ns=bar/app=hs2 +#### A masked pattern was here #### +537 +PREHOOK: query: select count(*) from logs2 where severity="INFO" +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-01-03/ns=foo/app=hs2 +PREHOOK: Input: default@logs2@dt=2019-11-23/ns=bar/app=hs2 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from logs2 where severity="INFO" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs2 +POSTHOOK: Input: default@logs2@dt=2019-01-03/ns=foo/app=hs2 +POSTHOOK: Input: default@logs2@dt=2019-11-23/ns=bar/app=hs2 +#### A masked pattern was here #### +1074 +PREHOOK: query: select count(*) from logs2 where severity="INFO" and app='hs2' +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-01-03/ns=foo/app=hs2 +PREHOOK: Input: default@logs2@dt=2019-11-23/ns=bar/app=hs2 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from logs2 where severity="INFO" and app='hs2' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs2 +POSTHOOK: Input: default@logs2@dt=2019-01-03/ns=foo/app=hs2 +POSTHOOK: Input: default@logs2@dt=2019-11-23/ns=bar/app=hs2 +#### A masked pattern was here #### +1074 +PREHOOK: query: select count(*) from logs2 where severity="INFO" and app='non-existent' +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from logs2 where severity="INFO" and app='non-existent' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs2 +#### A masked pattern was here #### +0 +PREHOOK: query: select count(*) from logs2 where ns='foo' +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-01-03/ns=foo/app=hs2 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from logs2 where ns='foo' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs2 +POSTHOOK: Input: default@logs2@dt=2019-01-03/ns=foo/app=hs2 +#### A masked pattern was here #### +806 +PREHOOK: query: select count(*) from logs2 where ns='bar' +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-11-23/ns=bar/app=hs2 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from logs2 where ns='bar' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs2 +POSTHOOK: Input: default@logs2@dt=2019-11-23/ns=bar/app=hs2 +#### A masked pattern was here #### +806 +PREHOOK: query: select count(*) from logs2 where unmatched is null and dt between '2019-01-03' and '2019-11-23' +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-01-03/ns=foo/app=hs2 +PREHOOK: Input: default@logs2@dt=2019-11-23/ns=bar/app=hs2 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from logs2 where unmatched is null and dt between '2019-01-03' and '2019-11-23' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs2 +POSTHOOK: Input: default@logs2@dt=2019-01-03/ns=foo/app=hs2 +POSTHOOK: Input: default@logs2@dt=2019-11-23/ns=bar/app=hs2 +#### A masked pattern was here #### +1438 +PREHOOK: query: select count(*) from logs2 where unmatched is not null +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-01-03/ns=foo/app=hs2 +PREHOOK: Input: default@logs2@dt=2019-11-23/ns=bar/app=hs2 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from logs2 where unmatched is not null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs2 +POSTHOOK: Input: default@logs2@dt=2019-01-03/ns=foo/app=hs2 +POSTHOOK: Input: default@logs2@dt=2019-11-23/ns=bar/app=hs2 +#### A masked pattern was here #### +174 +PREHOOK: query: select ts,severity,structured_data["thread"],structured_data["class"] from logs2 where severity="ERROR" order by ts +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-01-03/ns=foo/app=hs2 +PREHOOK: Input: default@logs2@dt=2019-11-23/ns=bar/app=hs2 +#### A masked pattern was here #### +POSTHOOK: query: select ts,severity,structured_data["thread"],structured_data["class"] from logs2 where severity="ERROR" order by ts +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs2 +POSTHOOK: Input: default@logs2@dt=2019-01-03/ns=foo/app=hs2 +POSTHOOK: Input: default@logs2@dt=2019-11-23/ns=bar/app=hs2 +#### A masked pattern was here #### +2019-03-21 07:07:09.857 ERROR HiveServer2-Background-Pool: Thread-85 metadata.Hive +2019-03-21 07:07:09.857 ERROR HiveServer2-Background-Pool: Thread-85 metadata.Hive +2019-03-21 07:07:17.035 ERROR HiveServer2-Background-Pool: Thread-92 ql.Driver +2019-03-21 07:07:17.035 ERROR HiveServer2-Background-Pool: Thread-92 ql.Driver +2019-03-21 07:07:17.105 ERROR HiveServer2-Background-Pool: Thread-92 operation.Operation +2019-03-21 07:07:17.105 ERROR HiveServer2-Background-Pool: Thread-92 operation.Operation +PREHOOK: query: select ts,severity,structured_data["thread"],structured_data["class"],app_name from logs2 where severity="WARN" order by ts +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-01-03/ns=foo/app=hs2 +PREHOOK: Input: default@logs2@dt=2019-11-23/ns=bar/app=hs2 +#### A masked pattern was here #### +POSTHOOK: query: select ts,severity,structured_data["thread"],structured_data["class"],app_name from logs2 where severity="WARN" order by ts +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs2 +POSTHOOK: Input: default@logs2@dt=2019-01-03/ns=foo/app=hs2 +POSTHOOK: Input: default@logs2@dt=2019-11-23/ns=bar/app=hs2 +#### A masked pattern was here #### +2019-03-21 07:06:15.136 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:15.136 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:15.137 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:15.137 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:15.137 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:15.137 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:15.284 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:15.284 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:15.284 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:15.284 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:15.284 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:15.284 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:18.555 WARN main session.SessionState hiveserver2 +2019-03-21 07:06:18.555 WARN main session.SessionState hiveserver2 +2019-03-21 07:06:19.798 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:19.798 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:19.798 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:19.798 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:19.798 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:19.798 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:20.142 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:20.142 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:20.143 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:20.143 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:20.143 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:20.143 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:20.143 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:20.143 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:20.405 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:20.405 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:20.405 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:20.405 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:20.405 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:20.405 WARN main conf.HiveConf hiveserver2 +2019-03-21 07:06:20.408 WARN main thrift.ThriftCLIService hiveserver2 +2019-03-21 07:06:20.408 WARN main thrift.ThriftCLIService hiveserver2 +2019-03-21 07:06:20.626 WARN main utils.ZKPaths hiveserver2 +2019-03-21 07:06:20.626 WARN main utils.ZKPaths hiveserver2 +2019-03-21 07:07:20.145 WARN NotificationEventPoll 0 conf.HiveConf hiveserver2 +2019-03-21 07:07:20.145 WARN NotificationEventPoll 0 conf.HiveConf hiveserver2 +2019-03-21 07:07:20.145 WARN NotificationEventPoll 0 conf.HiveConf hiveserver2 +2019-03-21 07:07:20.145 WARN NotificationEventPoll 0 conf.HiveConf hiveserver2 +2019-03-21 07:07:20.146 WARN NotificationEventPoll 0 conf.HiveConf hiveserver2 +2019-03-21 07:07:20.146 WARN NotificationEventPoll 0 conf.HiveConf hiveserver2 +PREHOOK: query: select decode(unmatched, 'UTF-8') from logs where unmatched is not null limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@logs +#### A masked pattern was here #### +POSTHOOK: query: select decode(unmatched, 'UTF-8') from logs where unmatched is not null limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@logs +#### A masked pattern was here #### ++ '[' hiveserver2 == data-load ']'ator ']' ++ '[' hiveserver2 == query-coordinator ']' ++ '[' hiveserver2 == query-executor ']' ++ . /fluentd-utils.sh +#### A masked pattern was here #### +++ set -xntd-utils.sh +PREHOOK: query: drop table logs +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@logs +PREHOOK: Output: default@logs +POSTHOOK: query: drop table logs +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@logs +POSTHOOK: Output: default@logs +PREHOOK: query: drop table logs2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@logs2 +PREHOOK: Output: default@logs2 +POSTHOOK: query: drop table logs2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@logs2 +POSTHOOK: Output: default@logs2 diff --git a/ql/src/test/results/clientpositive/llap/syslog_parser_file_pruning.q.out b/ql/src/test/results/clientpositive/llap/syslog_parser_file_pruning.q.out new file mode 100644 index 0000000000..2bb57a3f01 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/syslog_parser_file_pruning.q.out @@ -0,0 +1,676 @@ +PREHOOK: query: CREATE EXTERNAL TABLE logs2( +facility STRING, +severity STRING, +version STRING, +ts TIMESTAMP, +hostname STRING, +app_name STRING, +proc_id STRING, +msg_id STRING, +structured_data MAP, +msg BINARY, +unmatched BINARY +) +PARTITIONED BY(dt DATE,ns STRING,app STRING) +STORED BY 'org.apache.hadoop.hive.ql.log.syslog.SyslogStorageHandler' +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@logs2 +POSTHOOK: query: CREATE EXTERNAL TABLE logs2( +facility STRING, +severity STRING, +version STRING, +ts TIMESTAMP, +hostname STRING, +app_name STRING, +proc_id STRING, +msg_id STRING, +structured_data MAP, +msg BINARY, +unmatched BINARY +) +PARTITIONED BY(dt DATE,ns STRING,app STRING) +STORED BY 'org.apache.hadoop.hive.ql.log.syslog.SyslogStorageHandler' +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@logs2 +PREHOOK: query: MSCK REPAIR TABLE logs2 +PREHOOK: type: MSCK +PREHOOK: Output: default@logs2 +POSTHOOK: query: MSCK REPAIR TABLE logs2 +POSTHOOK: type: MSCK +POSTHOOK: Output: default@logs2 +Partitions not in metastore: logs2:dt=2019-03-21/ns=foo/app=hs2 logs2:dt=2019-03-22/ns=bar/app=hs2 +#### A masked pattern was here #### +PREHOOK: query: select severity,count(*) from logs2 where ts between '2019-03-21 07:00:00.0' and '2019-03-21 07:06:00.0' group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-03-21/ns=foo/app=hs2 +PREHOOK: Input: default@logs2@dt=2019-03-22/ns=bar/app=hs2 +#### A masked pattern was here #### +Stage-1 FILE SYSTEM COUNTERS: +Stage-1 HIVE COUNTERS: + DESERIALIZE_ERRORS: 0 + RECORDS_IN_Map_1: 331 + RECORDS_OUT_0: 0 + RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_9: 0 + RECORDS_OUT_OPERATOR_FS_14: 0 + RECORDS_OUT_OPERATOR_GBY_11: 0 + RECORDS_OUT_OPERATOR_GBY_13: 0 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_12: 0 + RECORDS_OUT_OPERATOR_SEL_10: 0 + RECORDS_OUT_OPERATOR_TS_0: 1137 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 2 + INPUT_FILES_Map_1: 2 + RAW_INPUT_SPLITS_Map_1: 2 +PREHOOK: query: select severity,count(*) from logs2 where ts between '2019-03-21 07:06:00.0' and '2019-03-21 07:07:00.0' group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-03-21/ns=foo/app=hs2 +PREHOOK: Input: default@logs2@dt=2019-03-22/ns=bar/app=hs2 +#### A masked pattern was here #### +Stage-1 FILE SYSTEM COUNTERS: +Stage-1 HIVE COUNTERS: + CREATED_FILES: 2 + DESERIALIZE_ERRORS: 0 + RECORDS_IN_Map_1: 331 + RECORDS_OUT_0: 3 + RECORDS_OUT_INTERMEDIATE_Map_1: 3 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_9: 445 + RECORDS_OUT_OPERATOR_FS_14: 3 + RECORDS_OUT_OPERATOR_GBY_11: 3 + RECORDS_OUT_OPERATOR_GBY_13: 3 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_12: 3 + RECORDS_OUT_OPERATOR_SEL_10: 445 + RECORDS_OUT_OPERATOR_TS_0: 1137 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 2 + INPUT_FILES_Map_1: 2 + RAW_INPUT_SPLITS_Map_1: 2 +DEBUG 50 +INFO 376 +WARN 19 +PREHOOK: query: select severity,count(*) from logs2 where ts between '2019-03-21 07:07:00.0' and '2019-03-21 07:08:00.0' group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-03-21/ns=foo/app=hs2 +PREHOOK: Input: default@logs2@dt=2019-03-22/ns=bar/app=hs2 +#### A masked pattern was here #### +Stage-1 FILE SYSTEM COUNTERS: +Stage-1 HIVE COUNTERS: + CREATED_FILES: 2 + DESERIALIZE_ERRORS: 0 + RECORDS_IN_Map_1: 331 + RECORDS_OUT_0: 4 + RECORDS_OUT_INTERMEDIATE_Map_1: 4 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_9: 274 + RECORDS_OUT_OPERATOR_FS_14: 4 + RECORDS_OUT_OPERATOR_GBY_11: 4 + RECORDS_OUT_OPERATOR_GBY_13: 4 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_12: 4 + RECORDS_OUT_OPERATOR_SEL_10: 274 + RECORDS_OUT_OPERATOR_TS_0: 1137 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 2 + INPUT_FILES_Map_1: 2 + RAW_INPUT_SPLITS_Map_1: 2 +DEBUG 107 +ERROR 3 +INFO 161 +WARN 3 +PREHOOK: query: select severity,count(*) from logs2 where ts between '2019-03-21 07:08:00.0' and '2019-03-21 08:08:00.0' group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-03-21/ns=foo/app=hs2 +PREHOOK: Input: default@logs2@dt=2019-03-22/ns=bar/app=hs2 +#### A masked pattern was here #### +Stage-1 FILE SYSTEM COUNTERS: +Stage-1 HIVE COUNTERS: + DESERIALIZE_ERRORS: 0 + RECORDS_IN_Map_1: 331 + RECORDS_OUT_0: 0 + RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_9: 0 + RECORDS_OUT_OPERATOR_FS_14: 0 + RECORDS_OUT_OPERATOR_GBY_11: 0 + RECORDS_OUT_OPERATOR_GBY_13: 0 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_12: 0 + RECORDS_OUT_OPERATOR_SEL_10: 0 + RECORDS_OUT_OPERATOR_TS_0: 1137 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 2 + INPUT_FILES_Map_1: 2 + RAW_INPUT_SPLITS_Map_1: 2 +PREHOOK: query: select severity,count(*) from logs2 where ts between '2019-03-21 07:00:00.0' and '2019-03-21 08:00:00.0' group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-03-21/ns=foo/app=hs2 +PREHOOK: Input: default@logs2@dt=2019-03-22/ns=bar/app=hs2 +#### A masked pattern was here #### +Stage-1 FILE SYSTEM COUNTERS: +Stage-1 HIVE COUNTERS: + CREATED_FILES: 2 + DESERIALIZE_ERRORS: 0 + RECORDS_IN_Map_1: 331 + RECORDS_OUT_0: 4 + RECORDS_OUT_INTERMEDIATE_Map_1: 4 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_9: 719 + RECORDS_OUT_OPERATOR_FS_14: 4 + RECORDS_OUT_OPERATOR_GBY_11: 4 + RECORDS_OUT_OPERATOR_GBY_13: 4 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_12: 4 + RECORDS_OUT_OPERATOR_SEL_10: 719 + RECORDS_OUT_OPERATOR_TS_0: 1137 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 2 + INPUT_FILES_Map_1: 2 + RAW_INPUT_SPLITS_Map_1: 2 +DEBUG 157 +ERROR 3 +INFO 537 +WARN 22 +PREHOOK: query: select severity,count(*) from logs2 where dt='2019-03-21' group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-03-21/ns=foo/app=hs2 +#### A masked pattern was here #### +Stage-1 FILE SYSTEM COUNTERS: +Stage-1 HIVE COUNTERS: + CREATED_FILES: 2 + DESERIALIZE_ERRORS: 0 + RECORDS_IN_Map_1: 806 + RECORDS_OUT_0: 5 + RECORDS_OUT_INTERMEDIATE_Map_1: 5 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_13: 5 + RECORDS_OUT_OPERATOR_GBY_10: 5 + RECORDS_OUT_OPERATOR_GBY_12: 5 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_11: 5 + RECORDS_OUT_OPERATOR_SEL_9: 806 + RECORDS_OUT_OPERATOR_TS_0: 806 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 +DEBUG 157 +ERROR 3 +INFO 537 +NULL 87 +WARN 22 +PREHOOK: query: select severity,count(*) from logs2 where ts between '2019-03-22 01:00:00.0' and '2019-03-22 01:08:00.0' group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-03-21/ns=foo/app=hs2 +PREHOOK: Input: default@logs2@dt=2019-03-22/ns=bar/app=hs2 +#### A masked pattern was here #### +Stage-1 FILE SYSTEM COUNTERS: +Stage-1 HIVE COUNTERS: + DESERIALIZE_ERRORS: 0 + RECORDS_IN_Map_1: 331 + RECORDS_OUT_0: 0 + RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_9: 0 + RECORDS_OUT_OPERATOR_FS_14: 0 + RECORDS_OUT_OPERATOR_GBY_11: 0 + RECORDS_OUT_OPERATOR_GBY_13: 0 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_12: 0 + RECORDS_OUT_OPERATOR_SEL_10: 0 + RECORDS_OUT_OPERATOR_TS_0: 1137 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 2 + INPUT_FILES_Map_1: 2 + RAW_INPUT_SPLITS_Map_1: 2 +PREHOOK: query: select severity,count(*) from logs2 where ts between '2019-03-22 01:08:00.0' and '2019-03-22 01:09:00.0' group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-03-21/ns=foo/app=hs2 +PREHOOK: Input: default@logs2@dt=2019-03-22/ns=bar/app=hs2 +#### A masked pattern was here #### +Stage-1 FILE SYSTEM COUNTERS: +Stage-1 HIVE COUNTERS: + CREATED_FILES: 2 + DESERIALIZE_ERRORS: 0 + RECORDS_IN_Map_1: 331 + RECORDS_OUT_0: 2 + RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_9: 220 + RECORDS_OUT_OPERATOR_FS_14: 2 + RECORDS_OUT_OPERATOR_GBY_11: 2 + RECORDS_OUT_OPERATOR_GBY_13: 2 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_12: 2 + RECORDS_OUT_OPERATOR_SEL_10: 220 + RECORDS_OUT_OPERATOR_TS_0: 1137 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 2 + INPUT_FILES_Map_1: 2 + RAW_INPUT_SPLITS_Map_1: 2 +DEBUG 31 +INFO 189 +PREHOOK: query: select severity,count(*) from logs2 where ts between '2019-03-22 01:09:00.0' and '2019-03-22 01:10:00.0' group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-03-21/ns=foo/app=hs2 +PREHOOK: Input: default@logs2@dt=2019-03-22/ns=bar/app=hs2 +#### A masked pattern was here #### +Stage-1 FILE SYSTEM COUNTERS: +Stage-1 HIVE COUNTERS: + DESERIALIZE_ERRORS: 0 + RECORDS_IN_Map_1: 331 + RECORDS_OUT_0: 0 + RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_9: 0 + RECORDS_OUT_OPERATOR_FS_14: 0 + RECORDS_OUT_OPERATOR_GBY_11: 0 + RECORDS_OUT_OPERATOR_GBY_13: 0 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_12: 0 + RECORDS_OUT_OPERATOR_SEL_10: 0 + RECORDS_OUT_OPERATOR_TS_0: 1137 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 2 + INPUT_FILES_Map_1: 2 + RAW_INPUT_SPLITS_Map_1: 2 +PREHOOK: query: select severity,count(*) from logs2 where ts between '2019-03-22 01:00:00.0' and '2019-03-22 02:00:00.0' group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-03-21/ns=foo/app=hs2 +PREHOOK: Input: default@logs2@dt=2019-03-22/ns=bar/app=hs2 +#### A masked pattern was here #### +Stage-1 FILE SYSTEM COUNTERS: +Stage-1 HIVE COUNTERS: + CREATED_FILES: 2 + DESERIALIZE_ERRORS: 0 + RECORDS_IN_Map_1: 331 + RECORDS_OUT_0: 2 + RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_9: 220 + RECORDS_OUT_OPERATOR_FS_14: 2 + RECORDS_OUT_OPERATOR_GBY_11: 2 + RECORDS_OUT_OPERATOR_GBY_13: 2 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_12: 2 + RECORDS_OUT_OPERATOR_SEL_10: 220 + RECORDS_OUT_OPERATOR_TS_0: 1137 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 2 + INPUT_FILES_Map_1: 2 + RAW_INPUT_SPLITS_Map_1: 2 +DEBUG 31 +INFO 189 +PREHOOK: query: select severity,count(*) from logs2 where dt='2019-03-22' group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-03-22/ns=bar/app=hs2 +#### A masked pattern was here #### +Stage-1 FILE SYSTEM COUNTERS: +Stage-1 HIVE COUNTERS: + CREATED_FILES: 2 + DESERIALIZE_ERRORS: 0 + RECORDS_IN_Map_1: 331 + RECORDS_OUT_0: 3 + RECORDS_OUT_INTERMEDIATE_Map_1: 3 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_13: 3 + RECORDS_OUT_OPERATOR_GBY_10: 3 + RECORDS_OUT_OPERATOR_GBY_12: 3 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_11: 3 + RECORDS_OUT_OPERATOR_SEL_9: 331 + RECORDS_OUT_OPERATOR_TS_0: 331 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 +DEBUG 31 +INFO 189 +NULL 111 +PREHOOK: query: select severity,count(*) from logs2 where ts between '2019-03-21 07:00:00.0' and '2019-03-21 07:06:00.0' group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-03-21/ns=foo/app=hs2 +PREHOOK: Input: default@logs2@dt=2019-03-22/ns=bar/app=hs2 +#### A masked pattern was here #### +Stage-1 FILE SYSTEM COUNTERS: +Stage-1 HIVE COUNTERS: + DESERIALIZE_ERRORS: 0 + RECORDS_IN_Map_1: 806 + RECORDS_OUT_0: 0 + RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_9: 0 + RECORDS_OUT_OPERATOR_FS_14: 0 + RECORDS_OUT_OPERATOR_GBY_11: 0 + RECORDS_OUT_OPERATOR_GBY_13: 0 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_12: 0 + RECORDS_OUT_OPERATOR_SEL_10: 0 + RECORDS_OUT_OPERATOR_TS_0: 806 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 2 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 +PREHOOK: query: select severity,count(*) from logs2 where ts between '2019-03-21 07:06:00.0' and '2019-03-21 07:07:00.0' group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-03-21/ns=foo/app=hs2 +PREHOOK: Input: default@logs2@dt=2019-03-22/ns=bar/app=hs2 +#### A masked pattern was here #### +Stage-1 FILE SYSTEM COUNTERS: +Stage-1 HIVE COUNTERS: + CREATED_FILES: 2 + DESERIALIZE_ERRORS: 0 + RECORDS_IN_Map_1: 806 + RECORDS_OUT_0: 3 + RECORDS_OUT_INTERMEDIATE_Map_1: 3 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_9: 445 + RECORDS_OUT_OPERATOR_FS_14: 3 + RECORDS_OUT_OPERATOR_GBY_11: 3 + RECORDS_OUT_OPERATOR_GBY_13: 3 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_12: 3 + RECORDS_OUT_OPERATOR_SEL_10: 445 + RECORDS_OUT_OPERATOR_TS_0: 806 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 2 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 +DEBUG 50 +INFO 376 +WARN 19 +PREHOOK: query: select severity,count(*) from logs2 where ts between '2019-03-21 07:07:00.0' and '2019-03-21 07:08:00.0' group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-03-21/ns=foo/app=hs2 +PREHOOK: Input: default@logs2@dt=2019-03-22/ns=bar/app=hs2 +#### A masked pattern was here #### +Stage-1 FILE SYSTEM COUNTERS: +Stage-1 HIVE COUNTERS: + CREATED_FILES: 2 + DESERIALIZE_ERRORS: 0 + RECORDS_IN_Map_1: 806 + RECORDS_OUT_0: 4 + RECORDS_OUT_INTERMEDIATE_Map_1: 4 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_9: 274 + RECORDS_OUT_OPERATOR_FS_14: 4 + RECORDS_OUT_OPERATOR_GBY_11: 4 + RECORDS_OUT_OPERATOR_GBY_13: 4 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_12: 4 + RECORDS_OUT_OPERATOR_SEL_10: 274 + RECORDS_OUT_OPERATOR_TS_0: 806 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 2 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 +DEBUG 107 +ERROR 3 +INFO 161 +WARN 3 +PREHOOK: query: select severity,count(*) from logs2 where ts between '2019-03-21 07:08:00.0' and '2019-03-21 08:08:00.0' group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-03-21/ns=foo/app=hs2 +PREHOOK: Input: default@logs2@dt=2019-03-22/ns=bar/app=hs2 +#### A masked pattern was here #### +Stage-1 FILE SYSTEM COUNTERS: +Stage-1 HIVE COUNTERS: + DESERIALIZE_ERRORS: 0 + RECORDS_IN_Map_1: 806 + RECORDS_OUT_0: 0 + RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_9: 0 + RECORDS_OUT_OPERATOR_FS_14: 0 + RECORDS_OUT_OPERATOR_GBY_11: 0 + RECORDS_OUT_OPERATOR_GBY_13: 0 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_12: 0 + RECORDS_OUT_OPERATOR_SEL_10: 0 + RECORDS_OUT_OPERATOR_TS_0: 806 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 2 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 +PREHOOK: query: select severity,count(*) from logs2 where ts between '2019-03-21 07:00:00.0' and '2019-03-21 08:00:00.0' group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-03-21/ns=foo/app=hs2 +PREHOOK: Input: default@logs2@dt=2019-03-22/ns=bar/app=hs2 +#### A masked pattern was here #### +Stage-1 FILE SYSTEM COUNTERS: +Stage-1 HIVE COUNTERS: + CREATED_FILES: 2 + DESERIALIZE_ERRORS: 0 + RECORDS_IN_Map_1: 806 + RECORDS_OUT_0: 4 + RECORDS_OUT_INTERMEDIATE_Map_1: 4 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_9: 719 + RECORDS_OUT_OPERATOR_FS_14: 4 + RECORDS_OUT_OPERATOR_GBY_11: 4 + RECORDS_OUT_OPERATOR_GBY_13: 4 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_12: 4 + RECORDS_OUT_OPERATOR_SEL_10: 719 + RECORDS_OUT_OPERATOR_TS_0: 806 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 2 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 +DEBUG 157 +ERROR 3 +INFO 537 +WARN 22 +PREHOOK: query: select severity,count(*) from logs2 where dt='2019-03-21' group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-03-21/ns=foo/app=hs2 +#### A masked pattern was here #### +Stage-1 FILE SYSTEM COUNTERS: +Stage-1 HIVE COUNTERS: + CREATED_FILES: 2 + DESERIALIZE_ERRORS: 0 + RECORDS_IN_Map_1: 806 + RECORDS_OUT_0: 5 + RECORDS_OUT_INTERMEDIATE_Map_1: 5 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_13: 5 + RECORDS_OUT_OPERATOR_GBY_10: 5 + RECORDS_OUT_OPERATOR_GBY_12: 5 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_11: 5 + RECORDS_OUT_OPERATOR_SEL_9: 806 + RECORDS_OUT_OPERATOR_TS_0: 806 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 +DEBUG 157 +ERROR 3 +INFO 537 +NULL 87 +WARN 22 +PREHOOK: query: select severity,count(*) from logs2 where ts between '2019-03-22 01:00:00.0' and '2019-03-22 01:08:00.0' group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-03-21/ns=foo/app=hs2 +PREHOOK: Input: default@logs2@dt=2019-03-22/ns=bar/app=hs2 +#### A masked pattern was here #### +Stage-1 FILE SYSTEM COUNTERS: +Stage-1 HIVE COUNTERS: + DESERIALIZE_ERRORS: 0 + RECORDS_IN_Map_1: 331 + RECORDS_OUT_0: 0 + RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_9: 0 + RECORDS_OUT_OPERATOR_FS_14: 0 + RECORDS_OUT_OPERATOR_GBY_11: 0 + RECORDS_OUT_OPERATOR_GBY_13: 0 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_12: 0 + RECORDS_OUT_OPERATOR_SEL_10: 0 + RECORDS_OUT_OPERATOR_TS_0: 331 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 2 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 +PREHOOK: query: select severity,count(*) from logs2 where ts between '2019-03-22 01:08:00.0' and '2019-03-22 01:09:00.0' group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-03-21/ns=foo/app=hs2 +PREHOOK: Input: default@logs2@dt=2019-03-22/ns=bar/app=hs2 +#### A masked pattern was here #### +Stage-1 FILE SYSTEM COUNTERS: +Stage-1 HIVE COUNTERS: + CREATED_FILES: 2 + DESERIALIZE_ERRORS: 0 + RECORDS_IN_Map_1: 331 + RECORDS_OUT_0: 2 + RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_9: 220 + RECORDS_OUT_OPERATOR_FS_14: 2 + RECORDS_OUT_OPERATOR_GBY_11: 2 + RECORDS_OUT_OPERATOR_GBY_13: 2 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_12: 2 + RECORDS_OUT_OPERATOR_SEL_10: 220 + RECORDS_OUT_OPERATOR_TS_0: 331 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 2 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 +DEBUG 31 +INFO 189 +PREHOOK: query: select severity,count(*) from logs2 where ts between '2019-03-22 01:09:00.0' and '2019-03-22 01:10:00.0' group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-03-21/ns=foo/app=hs2 +PREHOOK: Input: default@logs2@dt=2019-03-22/ns=bar/app=hs2 +#### A masked pattern was here #### +Stage-1 FILE SYSTEM COUNTERS: +Stage-1 HIVE COUNTERS: + DESERIALIZE_ERRORS: 0 + RECORDS_IN_Map_1: 331 + RECORDS_OUT_0: 0 + RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_9: 0 + RECORDS_OUT_OPERATOR_FS_14: 0 + RECORDS_OUT_OPERATOR_GBY_11: 0 + RECORDS_OUT_OPERATOR_GBY_13: 0 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_12: 0 + RECORDS_OUT_OPERATOR_SEL_10: 0 + RECORDS_OUT_OPERATOR_TS_0: 331 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 2 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 +PREHOOK: query: select severity,count(*) from logs2 where ts between '2019-03-22 01:00:00.0' and '2019-03-22 02:00:00.0' group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-03-21/ns=foo/app=hs2 +PREHOOK: Input: default@logs2@dt=2019-03-22/ns=bar/app=hs2 +#### A masked pattern was here #### +Stage-1 FILE SYSTEM COUNTERS: +Stage-1 HIVE COUNTERS: + CREATED_FILES: 2 + DESERIALIZE_ERRORS: 0 + RECORDS_IN_Map_1: 331 + RECORDS_OUT_0: 2 + RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_9: 220 + RECORDS_OUT_OPERATOR_FS_14: 2 + RECORDS_OUT_OPERATOR_GBY_11: 2 + RECORDS_OUT_OPERATOR_GBY_13: 2 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_12: 2 + RECORDS_OUT_OPERATOR_SEL_10: 220 + RECORDS_OUT_OPERATOR_TS_0: 331 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 2 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 +DEBUG 31 +INFO 189 +PREHOOK: query: select severity,count(*) from logs2 where dt='2019-03-22' group by severity +PREHOOK: type: QUERY +PREHOOK: Input: default@logs2 +PREHOOK: Input: default@logs2@dt=2019-03-22/ns=bar/app=hs2 +#### A masked pattern was here #### +Stage-1 FILE SYSTEM COUNTERS: +Stage-1 HIVE COUNTERS: + CREATED_FILES: 2 + DESERIALIZE_ERRORS: 0 + RECORDS_IN_Map_1: 331 + RECORDS_OUT_0: 3 + RECORDS_OUT_INTERMEDIATE_Map_1: 3 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_13: 3 + RECORDS_OUT_OPERATOR_GBY_10: 3 + RECORDS_OUT_OPERATOR_GBY_12: 3 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_11: 3 + RECORDS_OUT_OPERATOR_SEL_9: 331 + RECORDS_OUT_OPERATOR_TS_0: 331 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 +DEBUG 31 +INFO 189 +NULL 111 +PREHOOK: query: drop table logs2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@logs2 +PREHOOK: Output: default@logs2 diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/MetastoreSchemaTool.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/MetastoreSchemaTool.java index eafe0c6d46..27b0483c01 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/MetastoreSchemaTool.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/MetastoreSchemaTool.java @@ -439,6 +439,8 @@ public int run(String metastoreHome, String[] args, OptionGroup additionalOption task = new SchemaToolTaskCreateUser(); } else if (cmdLine.hasOption("dropAllDatabases")) { task = new SchemaToolTaskDrop(); + } else if (cmdLine.hasOption("createLogsTable")) { + task = new SchemaToolTaskCreateLogsTable(); } else { throw new HiveMetaException("No task defined!"); } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolCommandLine.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolCommandLine.java index 6282078411..a231df51e0 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolCommandLine.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolCommandLine.java @@ -72,6 +72,10 @@ private Options createOptions(OptionGroup additionalOptions) { .create("moveTable"); Option createUserOpt = new Option("createUser", "Create the Hive user, set hiveUser to the db" + " admin user and the hive password to the db admin password with this"); + Option createLogsTable = OptionBuilder + .hasArg() + .withDescription("Create table for Hive warehouse/compute logs") + .create("createLogsTable"); OptionGroup optGroup = new OptionGroup(); optGroup @@ -88,7 +92,8 @@ private Options createOptions(OptionGroup additionalOptions) { .addOption(alterCatalog) .addOption(moveDatabase) .addOption(moveTable) - .addOption(createUserOpt); + .addOption(createUserOpt) + .addOption(createLogsTable); optGroup.setRequired(true); Option userNameOpt = OptionBuilder.withArgName("user") @@ -165,6 +170,9 @@ private Options createOptions(OptionGroup additionalOptions) { .withDescription("Database a moving table is going to. This is " + "required if you are moving a table.") .create("toDatabase"); + Option retentionPeriod = OptionBuilder.hasArg() + .withDescription("Specify logs table retention period") + .create("retentionPeriod"); Options options = new Options(); options.addOption(help); @@ -190,6 +198,7 @@ private Options createOptions(OptionGroup additionalOptions) { options.addOption(hivePasswdOpt); options.addOption(hiveDbOpt); options.addOption(yesOpt); + options.addOption(retentionPeriod); if (additionalOptions != null) options.addOptionGroup(additionalOptions); return options; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolTaskCreateLogsTable.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolTaskCreateLogsTable.java new file mode 100644 index 0000000000..eb603e88f8 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolTaskCreateLogsTable.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.tools.schematool; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; + +import org.apache.hadoop.hive.metastore.HiveMetaException; + +/** Generate and run script to create logs table in the SYS schema at the specified location. */ +public class SchemaToolTaskCreateLogsTable extends SchemaToolTask { + /** Path of the warehouse/compute logs directory. */ + private String logPath; + private String retentionPeriod; + + @Override + void setCommandLineArguments(SchemaToolCommandLine cl) { + logPath = cl.getOptionValue("createLogsTable"); + retentionPeriod = cl.getOptionValue("retentionPeriod") == null ? "7d" : cl.getOptionValue("retentionPeriod"); + } + + @Override + void execute() throws HiveMetaException { + schemaTool.testConnectionToMetastore(); + + System.out.println("Starting creation of logs table"); + + File scriptFile = generateLogsTableScript(); + + String initScriptDir = scriptFile.getParent(); + String initScriptFile = scriptFile.getName(); + + try { + System.out.println("Initialization script " + initScriptFile); + if (!schemaTool.isDryRun()) { + schemaTool.execSql(initScriptDir, initScriptFile); + System.out.println("Initialization script completed"); + } + } catch (IOException e) { + throw new HiveMetaException("Logs table creation FAILED!", e); + } + } + + private File generateLogsTableScript() throws HiveMetaException { + try { + File tmpFile = File.createTempFile("schematool", ".sql"); + tmpFile.deleteOnExit(); + FileWriter fstream = new FileWriter(tmpFile.getPath()); + try (BufferedWriter out = new BufferedWriter(fstream)) { + out.write("USE SYS;" + System.getProperty("line.separator")); + + out.write("CREATE EXTERNAL TABLE logs"); + out.write(" (facility STRING, severity STRING,"); + out.write(" version STRING, ts TIMESTAMP, hostname STRING, app_name STRING,"); + out.write(" proc_id STRING, msg_id STRING, structured_data map, msg BINARY,"); + out.write(" unmatched BINARY)"); + out.write(" PARTITIONED BY (dt DATE, ns STRING, app STRING)"); + out.write(" STORED BY 'org.apache.hadoop.hive.ql.log.syslog.SyslogStorageHandler'"); + out.write(" LOCATION '" + logPath + "'"); + out.write(" TBLPROPERTIES (\"partition.retention.period\"=\"" + retentionPeriod + "\");"); + out.write(System.getProperty("line.separator")); + } + + return tmpFile; + } catch (Exception err) { + throw new HiveMetaException("Error generating logs table script", err); + } + } +}