diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 0880a960e6..c35d26dab6 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1878,7 +1878,6 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "in the number of rows filtered by a certain operator, which in turn might lead to overprovision or\n" + "underprovision of resources. This factor is applied to the cardinality estimation of IN clauses in\n" + "filter operators."), - // Concurrency HIVE_SUPPORT_CONCURRENCY("hive.support.concurrency", false, "Whether Hive supports concurrency control or not. \n" + @@ -3683,22 +3682,27 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal HIVE_CONF_INTERNAL_VARIABLE_LIST("hive.conf.internal.variable.list", "hive.added.files.path,hive.added.jars.path,hive.added.archives.path", "Comma separated list of variables which are used internally and should not be configurable."), - HIVE_SPARK_RSC_CONF_LIST("hive.spark.rsc.conf.list", SPARK_OPTIMIZE_SHUFFLE_SERDE.varname + "," + SPARK_CLIENT_FUTURE_TIMEOUT.varname, "Comma separated list of variables which are related to remote spark context.\n" + "Changing these variables will result in re-creating the spark session."), - HIVE_QUERY_TIMEOUT_SECONDS("hive.query.timeout.seconds", "0s", new TimeValidator(TimeUnit.SECONDS), "Timeout for Running Query in seconds. A nonpositive value means infinite. " + "If the query timeout is also set by thrift API call, the smaller one will be taken."), - - HIVE_EXEC_INPUT_LISTING_MAX_THREADS("hive.exec.input.listing.max.threads", 0, new SizeValidator(0L, true, 1024L, true), "Maximum number of threads that Hive uses to list file information from file systems (recommended > 1 for blobstore)."), + HIVE_QUERY_REEXECUTION_ENABLED("hive.query.reexecution.enabled", true, + "Enable query reexecutions"), + HIVE_QUERY_REEXECUTION_STRATEGIES("hive.query.reexecution.strategies", "overlay,reoptimize", + "comma separated list of plugin can be used:\n" + + " overlay: hiveconf subtree 'reexec.overlay' is used as an overlay in case of an execution errors out\n" + + " reoptimize: collects operator statistics during execution and recompile the query after a failure"), + HIVE_QUERY_MAX_REEXECUTION_COUNT("hive.query.reexecution.max.count", 1, + "Maximum number of re-executions for a single query."), + HIVE_QUERY_RESULTS_CACHE_ENABLED("hive.query.results.cache.enabled", true, "If the query results cache is enabled. This will keep results of previously executed queries " + "to be reused if the same query is executed again."), @@ -5082,4 +5086,23 @@ public static String generateMrDeprecationWarning() { return reverseMap; } } + + public void verifyAndSetAll(Map overlay) { + for (Entry entry : overlay.entrySet()) { + verifyAndSet(entry.getKey(), entry.getValue()); + } + } + + public Map subtree(String string) { + Map ret = new HashMap<>(); + for (Entry entry : getProps().entrySet()) { + String key = (String) entry.getKey(); + String value = (String) entry.getValue(); + if (key.startsWith(string)) { + ret.put(key.substring(string.length() + 1), value); + } + } + return ret; + } + } diff --git data/conf/hive-site.xml data/conf/hive-site.xml index b56cbd2469..9822760586 100644 --- data/conf/hive-site.xml +++ data/conf/hive-site.xml @@ -333,4 +333,9 @@ false + + hive.query.reexecution.enabled + false + + diff --git data/conf/llap/hive-site.xml data/conf/llap/hive-site.xml index c4c299c5de..8d53a05b0c 100644 --- data/conf/llap/hive-site.xml +++ data/conf/llap/hive-site.xml @@ -353,4 +353,14 @@ false + + tez.counters.max + 1024 + + + + hive.query.reexecution.enabled + false + + diff --git data/conf/perf-reg/spark/hive-site.xml data/conf/perf-reg/spark/hive-site.xml index 5ca660d4e7..c57eb4645e 100644 --- data/conf/perf-reg/spark/hive-site.xml +++ data/conf/perf-reg/spark/hive-site.xml @@ -270,4 +270,9 @@ false + + hive.query.reexecution.enabled + false + + diff --git data/conf/perf-reg/tez/hive-site.xml data/conf/perf-reg/tez/hive-site.xml index 62ecb74f3f..8a7534056c 100644 --- data/conf/perf-reg/tez/hive-site.xml +++ data/conf/perf-reg/tez/hive-site.xml @@ -287,4 +287,9 @@ false + + hive.query.reexecution.enabled + false + + diff --git data/conf/rlist/hive-site.xml data/conf/rlist/hive-site.xml index 630e481edc..7cd1896b70 100644 --- data/conf/rlist/hive-site.xml +++ data/conf/rlist/hive-site.xml @@ -324,4 +324,9 @@ false + + hive.query.reexecution.enabled + false + + diff --git data/conf/spark/local/hive-site.xml data/conf/spark/local/hive-site.xml index 8ff6256638..426a961dfc 100644 --- data/conf/spark/local/hive-site.xml +++ data/conf/spark/local/hive-site.xml @@ -266,4 +266,9 @@ false + + hive.query.reexecution.enabled + false + + diff --git data/conf/spark/standalone/hive-site.xml data/conf/spark/standalone/hive-site.xml index 84851c769c..02329689bb 100644 --- data/conf/spark/standalone/hive-site.xml +++ data/conf/spark/standalone/hive-site.xml @@ -271,4 +271,9 @@ false + + hive.query.reexecution.enabled + false + + diff --git data/conf/spark/yarn-client/hive-site.xml data/conf/spark/yarn-client/hive-site.xml index 6c633623ab..5de5aa637e 100644 --- data/conf/spark/yarn-client/hive-site.xml +++ data/conf/spark/yarn-client/hive-site.xml @@ -311,4 +311,9 @@ false + + hive.query.reexecution.enabled + false + + diff --git data/conf/tez/hive-site.xml data/conf/tez/hive-site.xml index 236adc7087..2cf7bc25b7 100644 --- data/conf/tez/hive-site.xml +++ data/conf/tez/hive-site.xml @@ -298,4 +298,9 @@ false + + hive.query.reexecution.enabled + false + + diff --git druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidScanQueryRecordReader.java druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidScanQueryRecordReader.java index cbeac2c00a..68ac88c09f 100644 --- druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidScanQueryRecordReader.java +++ druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidScanQueryRecordReader.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index 2776fe95f1..ff6769fc57 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -514,6 +514,9 @@ minillaplocal.query.files=\ bucketmapjoin6.q,\ bucketmapjoin7.q,\ bucketpruning1.q,\ + retry_failure.q,\ + retry_failure_stat_changes.q,\ + retry_failure_oom.q,\ bucketsortoptimize_insert_2.q,\ cbo_gby.q,\ cbo_join.q,\ diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 6cd7a136ae..d89bbce70a 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -1736,7 +1736,7 @@ private void maskPatterns(Pattern[] patterns, String fname) throws Exception { ppm.add(new PatternReplacementPair(Pattern.compile("\\{\"transactionid\":[1-9][0-9]*,\"bucketid\":"), "{\"transactionid\":### Masked txnid ###,\"bucketid\":")); - ppm.add(new PatternReplacementPair(Pattern.compile("attempt_[0-9]+"), "attempt_#ID#")); + ppm.add(new PatternReplacementPair(Pattern.compile("attempt_[0-9_]+"), "attempt_#ID#")); ppm.add(new PatternReplacementPair(Pattern.compile("vertex_[0-9_]+"), "vertex_#ID#")); ppm.add(new PatternReplacementPair(Pattern.compile("task_[0-9_]+"), "task_#ID#")); partialPlanMask = ppm.toArray(new PatternReplacementPair[ppm.size()]); diff --git itests/util/src/test/java/org/apache/hadoop/hive/ql/TestQTestUtil.java itests/util/src/test/java/org/apache/hadoop/hive/ql/TestQTestUtil.java index c01d87bf51..1a8eb33662 100644 --- itests/util/src/test/java/org/apache/hadoop/hive/ql/TestQTestUtil.java +++ itests/util/src/test/java/org/apache/hadoop/hive/ql/TestQTestUtil.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git ql/src/java/org/apache/hadoop/hive/ql/Context.java ql/src/java/org/apache/hadoop/hive/ql/Context.java index dba2dbb15b..58fa5f2287 100644 --- ql/src/java/org/apache/hadoop/hive/ql/Context.java +++ ql/src/java/org/apache/hadoop/hive/ql/Context.java @@ -28,6 +28,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Random; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -58,6 +59,10 @@ import org.apache.hadoop.hive.ql.parse.HiveParser; import org.apache.hadoop.hive.ql.parse.QB; import org.apache.hadoop.hive.ql.plan.LoadTableDesc; +import org.apache.hadoop.hive.ql.plan.mapper.EmptyStatsSource; +import org.apache.hadoop.hive.ql.plan.mapper.PlanMapper; +import org.apache.hadoop.hive.ql.plan.mapper.RuntimeStatsSource; +import org.apache.hadoop.hive.ql.plan.mapper.StatsSource; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.wm.WmContext; import org.apache.hadoop.hive.shims.ShimLoader; @@ -65,6 +70,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; + /** * Context for Semantic Analyzers. Usage: not reusable - construct a new one for * each query should call clear() at end of use to remove temporary folders @@ -153,6 +159,11 @@ private Operation operation = Operation.OTHER; private WmContext wmContext; + private boolean isExplainPlan = false; + private PlanMapper planMapper = new PlanMapper(); + private RuntimeStatsSource runtimeStatsSource; + private int executionIndex; + public void setOperation(Operation operation) { this.operation = operation; } @@ -229,7 +240,7 @@ public DestClausePrefix getDestNamePrefix(ASTNode curNode, QB queryBlock) { } if(!thisIsInASubquery) { throw new IllegalStateException("Expected '" + getMatchedText(curNode) + "' to be in sub-query or set operation."); - } + } return DestClausePrefix.INSERT; } switch (operation) { @@ -252,7 +263,7 @@ public DestClausePrefix getDestNamePrefix(ASTNode curNode, QB queryBlock) { assert insert != null && insert.getType() == HiveParser.TOK_INSERT; ASTNode query = (ASTNode) insert.getParent(); assert query != null && query.getType() == HiveParser.TOK_QUERY; - + for(int childIdx = 1; childIdx < query.getChildCount(); childIdx++) {//1st child is TOK_FROM assert query.getChild(childIdx).getType() == HiveParser.TOK_INSERT; if(insert == query.getChild(childIdx)) { @@ -997,7 +1008,7 @@ public void setSkipTableMasking(boolean skipTableMasking) { public ExplainConfiguration getExplainConfig() { return explainConfig; } - private boolean isExplainPlan = false; + public boolean isExplainPlan() { return isExplainPlan; } @@ -1033,4 +1044,33 @@ public void setIsLoadingMaterializedView(boolean isLoadingMaterializedView) { public String getExecutionId() { return executionId; } + + public PlanMapper getPlanMapper() { + return planMapper; + } + + public void setRuntimeStatsSource(RuntimeStatsSource runtimeStatsSource) { + this.runtimeStatsSource = runtimeStatsSource; + } + + public Optional getRuntimeStatsSource() { + return Optional.ofNullable(runtimeStatsSource); + } + + public StatsSource getStatsSource() { + if (runtimeStatsSource != null) { + return runtimeStatsSource; + } else { + // hierarchical; add def stats also here + return new EmptyStatsSource(); + } + } + + public int getExecutionIndex() { + return executionIndex; + } + + public void setExecutionIndex(int executionIndex) { + this.executionIndex = executionIndex; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/Driver.java ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 94999fed93..4054965569 100644 --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -76,6 +76,7 @@ import org.apache.hadoop.hive.ql.hooks.Entity; import org.apache.hadoop.hive.ql.hooks.HookContext; import org.apache.hadoop.hive.ql.hooks.HookUtils; +import org.apache.hadoop.hive.ql.hooks.PrivateHookContext; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.io.AcidUtils; @@ -108,6 +109,9 @@ import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.HiveOperation; import org.apache.hadoop.hive.ql.plan.TableDesc; +import org.apache.hadoop.hive.ql.plan.mapper.PlanMapper; +import org.apache.hadoop.hive.ql.plan.mapper.PlanMapperProcess; +import org.apache.hadoop.hive.ql.plan.mapper.RuntimeStatsSource; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.security.authorization.AuthorizationUtils; import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider; @@ -190,6 +194,7 @@ // Transaction manager used for the query. This will be set at compile time based on // either initTxnMgr or from the SessionState, in that order. private HiveTxnManager queryTxnMgr; + private RuntimeStatsSource runtimeStatsSource; private CacheUsage cacheUsage; private CacheEntry usedCacheEntry; @@ -281,6 +286,15 @@ public Schema getSchema() { return schema; } + @Override + public Context getContext() { + return ctx; + } + + public PlanMapper getPlanMapper() { + return ctx.getPlanMapper(); + } + /** * Get a Schema with fields represented with native Hive types */ @@ -556,6 +570,7 @@ public void run() { setTriggerContext(queryId); } + ctx.setRuntimeStatsSource(runtimeStatsSource); ctx.setCmd(command); ctx.setHDFSCleanup(true); @@ -578,7 +593,6 @@ public void run() { hookRunner.runBeforeCompileHook(command); perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.ANALYZE); - BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(queryState, tree); // Flush the metastore cache. This assures that we don't pick up objects from a previous // query running in this same thread. This has to be done after we get our semantic @@ -586,15 +600,7 @@ public void run() { // because at that point we need access to the objects. Hive.get().getMSC().flushCache(); - if(checkConcurrency() && startImplicitTxn(queryTxnMgr)) { - String userFromUGI = getUserFromUGI(); - if (!queryTxnMgr.isTxnOpen()) { - if(userFromUGI == null) { - throw createProcessorResponse(10); - } - long txnid = queryTxnMgr.openTxn(ctx, userFromUGI); - } - } + BaseSemanticAnalyzer sem; // Do semantic analysis and plan generation if (hookRunner.hasPreAnalyzeHooks()) { HiveSemanticAnalyzerHookContext hookCtx = new HiveSemanticAnalyzerHookContextImpl(); @@ -605,12 +611,15 @@ public void run() { hookCtx.setHiveOperation(queryState.getHiveOperation()); tree = hookRunner.runPreAnalyzeHooks(hookCtx, tree); - + sem = SemanticAnalyzerFactory.get(queryState, tree); + openTransaction(); sem.analyze(tree, ctx); hookCtx.update(sem); hookRunner.runPostAnalyzeHooks(hookCtx, sem.getAllRootTasks()); } else { + sem = SemanticAnalyzerFactory.get(queryState, tree); + openTransaction(); sem.analyze(tree, ctx); } LOG.info("Semantic Analysis Completed"); @@ -640,6 +649,8 @@ public void run() { plan.getFetchTask().initialize(queryState, plan, null, ctx.getOpContext()); } + PlanMapperProcess.runPostProcess(ctx.getPlanMapper()); + //do the authorization check if (!sem.skipAuthorization() && HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED)) { @@ -748,6 +759,18 @@ private void setTriggerContext(final String queryId) { ctx.setWmContext(wmContext); } + private void openTransaction() throws LockException, CommandProcessorResponse { + if (checkConcurrency() && startImplicitTxn(queryTxnMgr)) { + String userFromUGI = getUserFromUGI(); + if (!queryTxnMgr.isTxnOpen()) { + if (userFromUGI == null) { + throw createProcessorResponse(10); + } + long txnid = queryTxnMgr.openTxn(ctx, userFromUGI); + } + } + } + private boolean startImplicitTxn(HiveTxnManager txnManager) throws LockException { boolean shouldOpenImplicitTxn = !ctx.isExplainPlan(); //this is dumb. HiveOperation is not always set. see HIVE-16447/HIVE-16443 @@ -1919,9 +1942,9 @@ private void execute() throws CommandProcessorResponse { SessionState ss = SessionState.get(); - hookContext = new HookContext(plan, queryState, ctx.getPathToCS(), SessionState.get().getUserName(), + hookContext = new PrivateHookContext(plan, queryState, ctx.getPathToCS(), SessionState.get().getUserName(), ss.getUserIpAddress(), InetAddress.getLocalHost().getHostAddress(), operationId, - ss.getSessionId(), Thread.currentThread().getName(), ss.isHiveServerQuery(), perfLogger, queryInfo); + ss.getSessionId(), Thread.currentThread().getName(), ss.isHiveServerQuery(), perfLogger, queryInfo, ctx); hookContext.setHookType(HookContext.HookType.PRE_EXEC_HOOK); hookRunner.runPreHooks(hookContext); @@ -2588,4 +2611,9 @@ public QueryState getQueryState() { public HookRunner getHookRunner() { return hookRunner; } + + public void setRuntimeStatsSource(RuntimeStatsSource runtimeStatsSource) { + this.runtimeStatsSource = runtimeStatsSource; + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/DriverFactory.java ql/src/java/org/apache/hadoop/hive/ql/DriverFactory.java index 60e8de8fd4..0f6a80ef0d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/DriverFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/DriverFactory.java @@ -18,31 +18,54 @@ package org.apache.hadoop.hive.ql; +import java.util.ArrayList; + import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.ql.reexec.IReExecutionPlugin; +import org.apache.hadoop.hive.ql.reexec.ReExecDriver; +import org.apache.hadoop.hive.ql.reexec.ReExecutionOverlayPlugin; +import org.apache.hadoop.hive.ql.reexec.ReOptimizePlugin; + +import com.google.common.base.Strings; /** - * Constructs a driver for ql clients + * Constructs a driver for ql clients. */ public class DriverFactory { - enum ExecutionStrategy { - none { - @Override - IDriver build(QueryState queryState, String userName, QueryInfo queryInfo) { - return new Driver(queryState, userName, queryInfo); - } - }; - - abstract IDriver build(QueryState queryState, String userName, QueryInfo queryInfo); - } - public static IDriver newDriver(HiveConf conf) { return newDriver(getNewQueryState(conf), null, null); } public static IDriver newDriver(QueryState queryState, String userName, QueryInfo queryInfo) { - ExecutionStrategy strategy = ExecutionStrategy.none; - return strategy.build(queryState, userName, queryInfo); + boolean enabled = queryState.getConf().getBoolVar(ConfVars.HIVE_QUERY_REEXECUTION_ENABLED); + if (!enabled) { + return new Driver(queryState, userName, queryInfo); + } + + String strategies = queryState.getConf().getVar(ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES); + strategies = Strings.nullToEmpty(strategies).trim().toLowerCase(); + ArrayList plugins = new ArrayList<>(); + for (String string : strategies.split(",")) { + if (string.trim().isEmpty()) { + continue; + } + plugins.add(buildReExecPlugin(string)); + } + + return new ReExecDriver(queryState, userName, queryInfo, plugins); + } + + private static IReExecutionPlugin buildReExecPlugin(String name) throws RuntimeException { + if (name.equals("overlay")) { + return new ReExecutionOverlayPlugin(); + } + if (name.equals("reoptimize")) { + return new ReOptimizePlugin(); + } + throw new RuntimeException( + "Unknown re-execution plugin: " + name + " (" + ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES.varname + ")"); } private static QueryState getNewQueryState(HiveConf conf) { diff --git ql/src/java/org/apache/hadoop/hive/ql/HookRunner.java ql/src/java/org/apache/hadoop/hive/ql/HookRunner.java index 2a32a51588..a3105b631d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/HookRunner.java +++ ql/src/java/org/apache/hadoop/hive/ql/HookRunner.java @@ -318,4 +318,9 @@ public void addPostHook(ExecuteWithHookContext hook) { public void addOnFailureHook(ExecuteWithHookContext hook) { onFailureHooks.add(hook); } + + public void addSemanticAnalyzerHook(HiveSemanticAnalyzerHook hook) { + saHooks.add(hook); + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/IDriver.java ql/src/java/org/apache/hadoop/hive/ql/IDriver.java index 9f13fa8e88..41737fb423 100644 --- ql/src/java/org/apache/hadoop/hive/ql/IDriver.java +++ ql/src/java/org/apache/hadoop/hive/ql/IDriver.java @@ -21,6 +21,8 @@ import java.io.IOException; import java.util.List; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; +import org.apache.hadoop.hive.common.classification.InterfaceStability; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Schema; import org.apache.hadoop.hive.ql.exec.FetchTask; @@ -30,6 +32,8 @@ /** * Hive query executer driver */ +@InterfaceAudience.Private +@InterfaceStability.Unstable public interface IDriver extends CommandProcessor { int compile(String string); @@ -66,4 +70,7 @@ void destroy(); HiveConf getConf(); + + Context getContext(); + } diff --git ql/src/java/org/apache/hadoop/hive/ql/cache/results/CacheUsage.java ql/src/java/org/apache/hadoop/hive/ql/cache/results/CacheUsage.java index 08b791ad42..bb217a1fe8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/cache/results/CacheUsage.java +++ ql/src/java/org/apache/hadoop/hive/ql/cache/results/CacheUsage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java index 131127e50d..88a056b4e8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java +++ ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index 32fc257b03..67ad319a12 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -455,6 +455,7 @@ system.registerGenericUDF("array", GenericUDFArray.class); system.registerGenericUDF("assert_true", GenericUDFAssertTrue.class); + system.registerGenericUDF("assert_true_oom", GenericUDFAssertTrueOOM.class); system.registerGenericUDF("map", GenericUDFMap.class); system.registerGenericUDF("struct", GenericUDFStruct.class); system.registerGenericUDF("named_struct", GenericUDFNamedStruct.class); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewDesc.java ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewDesc.java index 1e28ca843f..f7af0737bf 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewDesc.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewTask.java index 2b345d6ec7..de120afbbc 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewTask.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java index 199b181290..c7a161044d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java @@ -84,7 +84,8 @@ protected final AtomicBoolean abortOp; private transient ExecMapperContext execContext; private transient boolean rootInitializeCalled = false; - protected transient long runTimeNumRows; + protected transient long numRows = 0; + protected transient long runTimeNumRows = 0; protected int indexForTezUnion = -1; private transient Configuration hconf; protected final transient Collection> asyncInitOperations = new HashSet<>(); @@ -108,6 +109,14 @@ // one of its parent is not in state CLOSE.. } + /** + * Counters. + */ + public enum Counter { + RECORDS_OUT_OPERATOR, + RECORDS_OUT_INTERMEDIATE + } + protected transient State state = State.UNINIT; private boolean useBucketizedHiveInputFormat; @@ -224,7 +233,6 @@ public RowSchema getSchema() { @SuppressWarnings("rawtypes") protected transient OutputCollector out; protected transient final Logger LOG = LoggerFactory.getLogger(getClass().getName()); - protected transient final Logger PLOG = LoggerFactory.getLogger(Operator.class.getName()); // for simple disabling logs from all operators protected transient String alias; protected transient Reporter reporter; protected String id; @@ -317,6 +325,7 @@ protected boolean areAllParentsInitialized() { @SuppressWarnings("unchecked") public final void initialize(Configuration hconf, ObjectInspector[] inputOIs) throws HiveException { + // String className = this.getClass().getName(); this.done = false; @@ -490,6 +499,14 @@ protected void initializeOp(Configuration hconf) throws HiveException { rootInitializeCalled = true; } + public String getCounterName(Counter counter, Configuration hconf) { + String context = hconf.get(Operator.CONTEXT_NAME_KEY, ""); + if (context != null && !context.isEmpty()) { + context = "_" + context.replace(" ", "_"); + } + return counter + context; + } + /** * Calls initialize on each of the children with outputObjetInspector as the * output row format. @@ -708,6 +725,10 @@ public void close(boolean abort) throws HiveException { if (conf != null && conf.getRuntimeStatsTmpDir() != null) { publishRunTimeStats(); } + LongWritable runTimeRowsWritable = new LongWritable(runTimeNumRows); + LongWritable recordCounter = new LongWritable(numRows); + statsMap.put(Counter.RECORDS_OUT_OPERATOR.name() + "_" + getOperatorId(), runTimeRowsWritable); + statsMap.put(getCounterName(Counter.RECORDS_OUT_INTERMEDIATE, hconf), recordCounter); this.runTimeNumRows = 0; reporter = null; @@ -970,12 +991,6 @@ private void baseForward(Object row, ObjectInspector rowInspector) } } - public void resetStats() { - for (String e : statsMap.keySet()) { - statsMap.get(e).set(0L); - } - } - public void reset(){ this.state=State.INIT; if (childOperators != null) { @@ -1562,4 +1577,22 @@ public boolean logicalEquals(Operator other) { (conf == other.getConf() || (conf != null && other.getConf() != null && conf.isSame(other.getConf()))); } + + // XXX: this could easily become a hot-spot + public final boolean logicalEqualsTree(Operator o) { + if (!logicalEquals(o)) { + return false; + } + if (o.getNumParent() != getNumParent()) { + return false; + } + for (int i = 0; i < getNumParent(); i++) { + Operator copL = parentOperators.get(i); + Operator copR = o.parentOperators.get(i); + if (!copL.logicalEquals(copR)) { + return false; + } + } + return true; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java index 395a5f450f..d4363fdf91 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java @@ -31,7 +31,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.CompilationOpContext; -import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.HiveKey; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; @@ -50,7 +49,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; import org.apache.hadoop.io.BinaryComparable; import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.OutputCollector; @@ -62,13 +60,6 @@ public class ReduceSinkOperator extends TerminalOperator implements Serializable, TopNHash.BinaryCollector { - /** - * Counters. - */ - public static enum Counter { - RECORDS_OUT_INTERMEDIATE - } - private static final long serialVersionUID = 1L; private static final MurmurHash hash = (MurmurHash) MurmurHash.getInstance(); @@ -140,10 +131,8 @@ // TODO: we only ever use one row of these at a time. Why do we need to cache multiple? protected transient Object[][] cachedKeys; - protected transient long numRows = 0; protected transient long cntr = 1; protected transient long logEveryNRows = 0; - private final transient LongWritable recordCounter = new LongWritable(); /** Kryo ctor. */ protected ReduceSinkOperator() { @@ -163,9 +152,6 @@ protected void initializeOp(Configuration hconf) throws HiveException { cntr = 1; logEveryNRows = HiveConf.getLongVar(hconf, HiveConf.ConfVars.HIVE_LOG_N_RECORDS); - final String vertexName = hconf.get(Operator.CONTEXT_NAME_KEY, ""); - statsMap.put(Utilities.getVertexCounterName(Counter.RECORDS_OUT_INTERMEDIATE.name(), vertexName), recordCounter); - List keys = conf.getKeyCols(); if (LOG.isDebugEnabled()) { @@ -363,7 +349,10 @@ public void process(Object row, int tag) throws HiveException { // if TopNHashes are active, proceed if not already excluded (i.e order by limit) final int firstIndex = (reducerHash != null) ? reducerHash.tryStoreKey(firstKey, partKeyNull) : TopNHash.FORWARD; - if (firstIndex == TopNHash.EXCLUDE) return; // Nothing to do. + if (firstIndex == TopNHash.EXCLUDE) + { + return; // Nothing to do. + } // Compute value and hashcode - we'd either store or forward them. BytesWritable value = makeValueWritable(row); @@ -531,6 +520,7 @@ protected void closeOp(boolean abort) throws HiveException { if (!abort && reducerHash != null) { reducerHash.flush(); } + runTimeNumRows = numRows; super.closeOp(abort); out = null; random = null; @@ -538,7 +528,6 @@ protected void closeOp(boolean abort) throws HiveException { if (LOG.isTraceEnabled()) { LOG.info(toString() + ": records written - " + numRows); } - recordCounter.set(numRows); } /** diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveInputCounters.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveInputCounters.java index 085d6a7d94..3100c338c4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveInputCounters.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveInputCounters.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/LlapObjectSubCache.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/LlapObjectSubCache.java index 0d31e6e422..7c5857c84a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/LlapObjectSubCache.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/LlapObjectSubCache.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkCommonOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkCommonOperator.java index 8dd7cfe58c..cc876d54c1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkCommonOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkCommonOperator.java @@ -28,7 +28,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator.Counter; import org.apache.hadoop.hive.ql.exec.TerminalOperator; import org.apache.hadoop.hive.ql.exec.TopNHash; import org.apache.hadoop.hive.ql.exec.Utilities; @@ -36,9 +35,7 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContextRegion; import org.apache.hadoop.hive.ql.exec.vector.VectorizationOperator; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.keyseries.VectorKeySeriesSerialized; import org.apache.hadoop.hive.ql.io.HiveKey; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.BaseWork; @@ -56,10 +53,7 @@ import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinarySerializeWrite; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.OutputCollector; -import org.apache.hive.common.util.HashCodeUtil; import com.google.common.base.Preconditions; @@ -136,10 +130,8 @@ // Where to write our key and value pairs. private transient OutputCollector out; - private transient long numRows = 0; private transient long cntr = 1; private transient long logEveryNRows = 0; - private final transient LongWritable recordCounter = new LongWritable(); // For debug tracing: the name of the map or reduce task. protected transient String taskName; @@ -274,7 +266,6 @@ protected void initializeOp(Configuration hconf) throws HiveException { if (context != null && !context.isEmpty()) { context = "_" + context.replace(" ","_"); } - statsMap.put(Counter.RECORDS_OUT_INTERMEDIATE + context, recordCounter); reduceSkipTag = conf.getSkipTag(); reduceTagByte = (byte) conf.getTag(); @@ -355,7 +346,9 @@ protected void collect(HiveKey keyWritable, BytesWritable valueWritable) final int firstIndex = reducerHash.tryStoreKey(keyWritable, /* partColsIsNull */ false); - if (firstIndex == TopNHash.EXCLUDE) return; // Nothing to do. + if (firstIndex == TopNHash.EXCLUDE) { + return; // Nothing to do. + } if (firstIndex == TopNHash.FORWARD) { doCollect(keyWritable, valueWritable); @@ -399,6 +392,7 @@ protected void closeOp(boolean abort) throws HiveException { if (!abort && reducerHash != null) { reducerHash.flush(); } + runTimeNumRows = numRows; super.closeOp(abort); out = null; reducerHash = null; @@ -406,7 +400,6 @@ protected void closeOp(boolean abort) throws HiveException { LOG.info(toString() + ": records written - " + numRows); } this.runTimeNumRows = numRows; - recordCounter.set(numRows); } /** diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkEmptyKeyOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkEmptyKeyOperator.java index 134fc0ff0b..78e64d8c16 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkEmptyKeyOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkEmptyKeyOperator.java @@ -18,54 +18,16 @@ package org.apache.hadoop.hive.ql.exec.vector.reducesink; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Properties; -import java.util.Random; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator.Counter; -import org.apache.hadoop.hive.ql.exec.TerminalOperator; -import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.exec.vector.VectorExtractRow; -import org.apache.hadoop.hive.ql.exec.vector.VectorSerializeRow; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; -import org.apache.hadoop.hive.ql.exec.vector.VectorizationContextRegion; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.keyseries.VectorKeySeriesSerialized; -import org.apache.hadoop.hive.ql.io.HiveKey; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.plan.BaseWork; import org.apache.hadoop.hive.ql.plan.OperatorDesc; -import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; -import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.plan.VectorDesc; -import org.apache.hadoop.hive.ql.plan.VectorReduceSinkDesc; -import org.apache.hadoop.hive.ql.plan.VectorReduceSinkInfo; -import org.apache.hadoop.hive.ql.plan.api.OperatorType; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.ByteStream.Output; -import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe; -import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableSerializeWrite; -import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinarySerializeWrite; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.mapred.OutputCollector; -import org.apache.hive.common.util.HashCodeUtil; import com.google.common.base.Preconditions; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkObjectHashOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkObjectHashOperator.java index 1eb72ce4d9..1bc3fdabac 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkObjectHashOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkObjectHashOperator.java @@ -18,54 +18,26 @@ package org.apache.hadoop.hive.ql.exec.vector.reducesink; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Properties; import java.util.Random; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator.Counter; -import org.apache.hadoop.hive.ql.exec.TerminalOperator; -import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.vector.VectorExtractRow; import org.apache.hadoop.hive.ql.exec.vector.VectorSerializeRow; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; -import org.apache.hadoop.hive.ql.exec.vector.VectorizationContextRegion; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.keyseries.VectorKeySeriesSerialized; -import org.apache.hadoop.hive.ql.io.HiveKey; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.plan.BaseWork; import org.apache.hadoop.hive.ql.plan.OperatorDesc; -import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; -import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.plan.VectorDesc; -import org.apache.hadoop.hive.ql.plan.VectorReduceSinkDesc; -import org.apache.hadoop.hive.ql.plan.VectorReduceSinkInfo; -import org.apache.hadoop.hive.ql.plan.api.OperatorType; -import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.ByteStream.Output; -import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe; import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableSerializeWrite; -import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinarySerializeWrite; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.mapred.OutputCollector; -import org.apache.hive.common.util.HashCodeUtil; import com.google.common.base.Preconditions; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkUniformHashOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkUniformHashOperator.java index 384bd74686..c98663c937 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkUniformHashOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkUniformHashOperator.java @@ -18,53 +18,18 @@ package org.apache.hadoop.hive.ql.exec.vector.reducesink; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Properties; -import java.util.Random; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.CompilationOpContext; -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator.Counter; -import org.apache.hadoop.hive.ql.exec.TerminalOperator; -import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.exec.vector.VectorExtractRow; -import org.apache.hadoop.hive.ql.exec.vector.VectorSerializeRow; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; -import org.apache.hadoop.hive.ql.exec.vector.VectorizationContextRegion; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.keyseries.VectorKeySeriesSerialized; -import org.apache.hadoop.hive.ql.io.HiveKey; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.plan.BaseWork; import org.apache.hadoop.hive.ql.plan.OperatorDesc; -import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; -import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.plan.VectorDesc; -import org.apache.hadoop.hive.ql.plan.VectorReduceSinkDesc; -import org.apache.hadoop.hive.ql.plan.VectorReduceSinkInfo; -import org.apache.hadoop.hive.ql.plan.api.OperatorType; -import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.ByteStream.Output; -import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe; -import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableSerializeWrite; -import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinarySerializeWrite; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.mapred.OutputCollector; import org.apache.hive.common.util.HashCodeUtil; import com.google.common.base.Preconditions; diff --git ql/src/java/org/apache/hadoop/hive/ql/hooks/PrivateHookContext.java ql/src/java/org/apache/hadoop/hive/ql/hooks/PrivateHookContext.java new file mode 100644 index 0000000000..605436b5f5 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/hooks/PrivateHookContext.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.hooks; + +import java.util.Map; + +import org.apache.hadoop.fs.ContentSummary; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; +import org.apache.hadoop.hive.common.classification.InterfaceStability; +import org.apache.hadoop.hive.ql.Context; +import org.apache.hadoop.hive.ql.QueryInfo; +import org.apache.hadoop.hive.ql.QueryPlan; +import org.apache.hadoop.hive.ql.QueryState; +import org.apache.hadoop.hive.ql.log.PerfLogger; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class PrivateHookContext extends HookContext { + + private final Context ctx; + + public PrivateHookContext(QueryPlan queryPlan, QueryState queryState, + Map inputPathToContentSummary, String userName, String ipAddress, + String hiveInstanceAddress, String operationId, String sessionId, String threadId, boolean isHiveServerQuery, + PerfLogger perfLogger, QueryInfo queryInfo, Context ctx) throws Exception { + super(queryPlan, queryState, inputPathToContentSummary, userName, ipAddress, hiveInstanceAddress, operationId, + sessionId, threadId, isHiveServerQuery, perfLogger, queryInfo); + this.ctx = ctx; + } + + public Context getContext() { + return ctx; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveException.java ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveException.java index b75850760f..bfbbe3a7d7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveException.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveException.java @@ -28,6 +28,7 @@ @InterfaceAudience.Public @InterfaceStability.Stable public class HiveException extends Exception { + /** * Standard predefined message with error code and possibly SQL State, etc. */ diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java index b0cf3bd94e..bc6d0bb2e0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java @@ -598,6 +598,7 @@ private static void gatherDPPTableScanOps( new LinkedList<>(tableToTotalSize.entrySet()); Collections.sort(sortedTables, Collections.reverseOrder( new Comparator>() { + @Override public int compare(Map.Entry o1, Map.Entry o2) { return (o1.getValue()).compareTo(o2.getValue()); } @@ -637,6 +638,7 @@ private static void gatherReduceSinkOpsByInput(Multimap, new LinkedList<>(opToTotalSize.entrySet()); Collections.sort(sortedOps, Collections.reverseOrder( new Comparator, Long>>() { + @Override public int compare(Map.Entry, Long> o1, Map.Entry, Long> o2) { int valCmp = o1.getValue().compareTo(o2.getValue()); if (valCmp == 0) { @@ -648,6 +650,7 @@ public int compare(Map.Entry, Long> o1, Map.Entry, Long> return sortedOps; } + // FIXME: probably this should also be integrated with isSame() logics private static boolean areMergeable(ParseContext pctx, SharedWorkOptimizerCache optimizerCache, TableScanOperator tsOp1, TableScanOperator tsOp2) throws SemanticException { // First we check if the two table scan operators can actually be merged diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOpMaterializationValidator.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOpMaterializationValidator.java index 8c1bcb3f62..df216e7555 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOpMaterializationValidator.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOpMaterializationValidator.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index 783a672c47..13a2fc478f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -28,21 +28,17 @@ import java.util.HashMap; import java.util.HashSet; import java.util.IdentityHashMap; -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Random; import java.util.Map.Entry; import java.util.Properties; import java.util.Set; import java.util.Stack; -import java.util.TreeMap; import java.util.TreeSet; import java.util.regex.Pattern; import org.apache.commons.lang.ArrayUtils; -import org.apache.calcite.util.Pair; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.hadoop.hive.ql.parse.spark.SparkPartitionPruningSinkOperator; import org.slf4j.Logger; @@ -83,16 +79,10 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationDesc; import org.apache.hadoop.hive.ql.exec.vector.VectorColumnOutputMapping; import org.apache.hadoop.hive.ql.exec.vector.VectorColumnSourceMapping; -import org.apache.hadoop.hive.ql.exec.vector.VectorFileSinkOperator; -import org.apache.hadoop.hive.ql.exec.vector.VectorFilterOperator; -import org.apache.hadoop.hive.ql.exec.vector.VectorGroupByOperator; import org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator; import org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOuterFilteredOperator; -import org.apache.hadoop.hive.ql.exec.vector.VectorReduceSinkOperator; -import org.apache.hadoop.hive.ql.exec.vector.VectorSelectOperator; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; import org.apache.hadoop.hive.ql.exec.vector.VectorizationOperator; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedBatchUtil; import org.apache.hadoop.hive.ql.exec.vector.VectorizedUDAFs; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext.HiveVectorAdaptorUsageMode; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext.InConstantType; @@ -105,22 +95,15 @@ import org.apache.hadoop.hive.ql.io.OneNullRowInputFormat; import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx; -import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; -import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; import org.apache.hadoop.hive.ql.lib.Dispatcher; -import org.apache.hadoop.hive.ql.lib.GraphWalker; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; -import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; -import org.apache.hadoop.hive.ql.lib.PreOrderOnceWalker; -import org.apache.hadoop.hive.ql.lib.PreOrderWalker; import org.apache.hadoop.hive.ql.lib.Rule; import org.apache.hadoop.hive.ql.lib.RuleRegExp; import org.apache.hadoop.hive.ql.lib.TaskGraphWalker; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowType; import org.apache.hadoop.hive.ql.plan.AbstractOperatorDesc; import org.apache.hadoop.hive.ql.plan.AggregationDesc; import org.apache.hadoop.hive.ql.plan.AppMasterEventDesc; @@ -138,11 +121,9 @@ import org.apache.hadoop.hive.ql.plan.MapJoinDesc; import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.ql.plan.MapredWork; -import org.apache.hadoop.hive.ql.plan.OpTraits; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.PTFDesc; import org.apache.hadoop.hive.ql.plan.SelectDesc; -import org.apache.hadoop.hive.ql.plan.Statistics; import org.apache.hadoop.hive.ql.plan.VectorAppMasterEventDesc; import org.apache.hadoop.hive.ql.plan.VectorDesc; import org.apache.hadoop.hive.ql.plan.VectorFileSinkDesc; @@ -179,13 +160,9 @@ import org.apache.hadoop.hive.ql.plan.VectorReduceSinkInfo; import org.apache.hadoop.hive.ql.plan.VectorPartitionDesc; import org.apache.hadoop.hive.ql.plan.VectorSelectDesc; -import org.apache.hadoop.hive.ql.plan.VectorReduceSinkDesc; -import org.apache.hadoop.hive.ql.plan.VectorReduceSinkInfo; -import org.apache.hadoop.hive.ql.plan.VectorPartitionDesc; import org.apache.hadoop.hive.ql.plan.api.OperatorType; import org.apache.hadoop.hive.ql.plan.ptf.OrderExpressionDef; import org.apache.hadoop.hive.ql.plan.ptf.PTFExpressionDef; -import org.apache.hadoop.hive.ql.plan.ptf.PartitionDef; import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; import org.apache.hadoop.hive.ql.plan.ptf.WindowFrameDef; import org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef; @@ -231,8 +208,6 @@ import org.apache.hadoop.hive.ql.udf.UDFWeekOfYear; import org.apache.hadoop.hive.ql.udf.UDFYear; import org.apache.hadoop.hive.ql.udf.generic.*; -import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator; -import org.apache.hadoop.hive.ql.udf.ptf.WindowingTableFunction; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.NullStructSerDe; @@ -256,8 +231,6 @@ import org.apache.hadoop.mapred.TextInputFormat; import org.apache.hive.common.util.AnnotationUtils; import org.apache.hadoop.util.ReflectionUtils; -import org.apache.hadoop.util.StringUtils; - import com.google.common.collect.ImmutableSet; import com.google.common.base.Preconditions; @@ -1415,7 +1388,9 @@ private boolean isInputFormatExcluded(String inputFileFormatClassName, Collectio return false; } for (Class badClass : excludes) { - if (badClass.isAssignableFrom(ifClass)) return true; + if (badClass.isAssignableFrom(ifClass)) { + return true; + } } return false; } @@ -4792,6 +4767,7 @@ private static VectorPTFInfo createVectorPTFInfo(Operator sigMap; + // FIXME: this is currently retained... + // but later the signature should be able to serve the same comparision granulaty level as op.logicalEquals right now + private Operator op; + + private OpSignature(Operator op) { + this.op = op; + sigMap = new HashMap<>(); + // FIXME: consider to operator info as well..not just conf? + SignatureUtils1.write(sigMap, op.getConf()); + } + + public static OpSignature of(Operator op) { + return new OpSignature(op); + } + + @Override + public int hashCode() { + return sigMap.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || !(obj instanceof OpSignature)) { + return false; + } + if (obj == this) { + return true; + } + OpSignature o = (OpSignature) obj; + return op.logicalEquals(o.op); + } + + public boolean signatureCompare(OpSignature other) { + return sigMap.equals(other.sigMap); + } + + @VisibleForTesting + public void proveEquals(OpSignature other) { + proveEquals(sigMap,other.sigMap); + } + + private static void proveEquals(Map m1, Map m2) { + for (Entry e : m1.entrySet()) { + String key = e.getKey(); + Object v1 = e.getValue(); + Object v2 = m2.get(key); + if (v1 == v2) { + continue; + } + if (v1 == null || v2 == null || !v1.equals(v2)) { + throw new RuntimeException(String.format("equals fails: %s (%s!=%s)", key, v1, v2)); + } + } + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/signature/OpTreeSignature.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/signature/OpTreeSignature.java new file mode 100644 index 0000000000..a45e2c9729 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/signature/OpTreeSignature.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.optimizer.signature; + +import java.util.ArrayList; +import java.util.Objects; + +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.plan.OperatorDesc; + +/** + * Operator tree signature. + */ +public class OpTreeSignature { + private Operator op; + private int hashCode; + private OpSignature sig; + private ArrayList parentSig; + + OpTreeSignature(Operator op, OpTreeSignatureFactory osf) { + this.op = op; + sig = OpSignature.of(op); + parentSig = new ArrayList<>(); + for (Operator parentOp : op.getParentOperators()) { + parentSig.add(osf.getSignature(parentOp)); + } + hashCode = Objects.hash(sig, parentSig); + } + + public static OpTreeSignature of(Operator root) { + return of(root, OpTreeSignatureFactory.DIRECT); + } + + public static OpTreeSignature of(Operator op, OpTreeSignatureFactory osf) { + return new OpTreeSignature(op, osf); + } + + @Override + public int hashCode() { + return hashCode; + } + + @Override + public boolean equals(Object obj) { + if (obj == null || !(obj instanceof OpTreeSignature)) { + return false; + } + if (obj == this) { + return true; + } + OpTreeSignature o = (OpTreeSignature) obj; + return logicalEqualsTree(op, o.op); + } + + // XXX: this is ain't cheap! :) + private final boolean logicalEqualsTree(Operator o1, Operator o) { + if (!o1.logicalEquals(o)) { + return false; + } + if (o.getNumParent() != o1.getNumParent()) { + return false; + } + for (int i = 0; i < o1.getNumParent(); i++) { + Operator copL = o1.getParentOperators().get(i); + Operator copR = o.getParentOperators().get(i); + if (!copL.logicalEquals(copR)) { + return false; + } + } + return true; + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/signature/OpTreeSignatureFactory.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/signature/OpTreeSignatureFactory.java new file mode 100644 index 0000000000..3df5ee946e --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/signature/OpTreeSignatureFactory.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.optimizer.signature; + +import java.util.IdentityHashMap; +import java.util.Map; + +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.plan.OperatorDesc; + +/** + * A simple cache backend to prevent repeated signature computations. + */ +public interface OpTreeSignatureFactory { + + public OpTreeSignature getSignature(Operator op); + + static final OpTreeSignatureFactory DIRECT = new Direct(); + + public static OpTreeSignatureFactory direct() { + return DIRECT; + } + + public static OpTreeSignatureFactory newCache() { + return new CachedFactory(); + } + + // FIXME: possible alternative: move both OpSignature/OpTreeSignature into + // under some class as nested ones; and that way this factory level caching can be made "transparent" + + static class Direct implements OpTreeSignatureFactory { + + @Override + public OpTreeSignature getSignature(Operator op) { + return OpTreeSignature.of(op, this); + } + + } + + static class CachedFactory implements OpTreeSignatureFactory { + + Map, OpTreeSignature> cache = new IdentityHashMap<>(); + + @Override + public OpTreeSignature getSignature(Operator op) { + return cache.computeIfAbsent(op, k -> OpTreeSignature.of(op, this)); + } + + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkPartitionPruningSinkDesc.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkPartitionPruningSinkDesc.java index d1c53cf345..1d798cf864 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkPartitionPruningSinkDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkPartitionPruningSinkDesc.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.ql.plan.OperatorDesc; +import org.apache.hadoop.hive.ql.plan.Signature; import org.apache.hadoop.hive.ql.plan.TableDesc; import java.util.ArrayList; @@ -134,6 +135,7 @@ public String displayTargetColumns() { info -> info.columnName + " (" + info.columnType + ")").toArray()); } + @Signature public TableDesc getTable() { return table; } @@ -156,4 +158,5 @@ public boolean isSame(OperatorDesc other) { } return false; } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java index 9a3f81c98f..6ac7efdbef 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java @@ -27,11 +27,12 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; import java.util.Set; import java.util.Stack; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.CommonJoinOperator; @@ -74,6 +75,8 @@ import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.Statistics; import org.apache.hadoop.hive.ql.plan.Statistics.State; +import org.apache.hadoop.hive.ql.plan.mapper.RuntimeStatsSource; +import org.apache.hadoop.hive.ql.stats.OperatorStats; import org.apache.hadoop.hive.ql.stats.StatsUtils; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; @@ -132,7 +135,9 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, try { // gather statistics for the first time and the attach it to table scan operator Statistics stats = StatsUtils.collectStatistics(aspCtx.getConf(), partList, colStatsCached, table, tsop); - tsop.setStatistics(stats.clone()); + + stats = applyRuntimeStats(aspCtx.getParseContext().getContext(), stats, tsop, stats); + tsop.setStatistics(stats); if (LOG.isDebugEnabled()) { LOG.debug("[0] STATS-" + tsop.toString() + " (" + table.getTableName() + "): " + @@ -144,6 +149,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } return null; } + } /** @@ -181,14 +187,15 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, if (satisfyPrecondition(parentStats)) { // this will take care of mapping between input column names and output column names. The // returned column stats will have the output column names. - List colStats = StatsUtils.getColStatisticsFromExprMap(conf, parentStats, - sop.getColumnExprMap(), sop.getSchema()); + List colStats = + StatsUtils.getColStatisticsFromExprMap(conf, parentStats, sop.getColumnExprMap(), sop.getSchema()); stats.setColumnStats(colStats); // in case of select(*) the data size does not change if (!sop.getConf().isSelectStar() && !sop.getConf().isSelStarNoCompute()) { long dataSize = StatsUtils.getDataSizeFromColumnStats(stats.getNumRows(), colStats); stats.setDataSize(dataSize); } + stats = applyRuntimeStats(aspCtx.getParseContext().getContext(), stats, sop, parentStats); sop.setStatistics(stats); if (LOG.isDebugEnabled()) { @@ -196,7 +203,8 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } } else { if (parentStats != null) { - sop.setStatistics(parentStats.clone()); + stats = applyRuntimeStats(aspCtx.getParseContext().getContext(), stats, sop, parentStats); + sop.setStatistics(stats); if (LOG.isDebugEnabled()) { LOG.debug("[1] STATS-" + sop.toString() + ": " + parentStats.extendedToString()); @@ -299,7 +307,10 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, LOG.debug("[1] STATS-" + fop.toString() + ": " + st.extendedToString()); } } + + st = applyRuntimeStats(aspCtx.getParseContext().getContext(), st, fop, parentStats); fop.setStatistics(st); + aspCtx.setAndExprStats(null); } return null; @@ -1249,6 +1260,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } } + stats = applyRuntimeStats(aspCtx.getParseContext().getContext(), stats, gop, parentStats); gop.setStatistics(stats); if (LOG.isDebugEnabled() && stats != null) { @@ -1576,6 +1588,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } } + stats = applyRuntimeStats(aspCtx.getParseContext().getContext(), stats, jop, stats); jop.setStatistics(stats); if (LOG.isDebugEnabled()) { @@ -1665,6 +1678,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } } + wcStats = applyRuntimeStats(aspCtx.getParseContext().getContext(), wcStats, jop, wcStats); jop.setStatistics(wcStats); if (LOG.isDebugEnabled()) { @@ -2215,6 +2229,7 @@ private long getDenominator(List distinctVals) { @Override public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { + AnnotateStatsProcCtx aspCtx = (AnnotateStatsProcCtx) procCtx; LimitOperator lop = (LimitOperator) nd; Operator parent = lop.getParentOperators().get(0); Statistics parentStats = parent.getStatistics(); @@ -2232,6 +2247,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, if (limit <= parentStats.getNumRows()) { updateStats(stats, limit, true, lop); } + stats = applyRuntimeStats(aspCtx.getParseContext().getContext(), stats, lop, parentStats); lop.setStatistics(stats); if (LOG.isDebugEnabled()) { @@ -2243,7 +2259,8 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, // in the absence of column statistics, compute data size based on // based on average row size limit = StatsUtils.getMaxIfOverflow(limit); - Statistics wcStats = parentStats.scaleToRowCount(limit); + Statistics wcStats = parentStats.scaleToRowCount(limit, true); + wcStats = applyRuntimeStats(aspCtx.getParseContext().getContext(), wcStats, lop, parentStats); lop.setStatistics(wcStats); if (LOG.isDebugEnabled()) { LOG.debug("[1] STATS-" + lop.toString() + ": " + wcStats.extendedToString()); @@ -2265,8 +2282,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, public static class ReduceSinkStatsRule extends DefaultStatsRule implements NodeProcessor { @Override - public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { + public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { ReduceSinkOperator rop = (ReduceSinkOperator) nd; Operator parent = rop.getParentOperators().get(0); Statistics parentStats = parent.getStatistics(); @@ -2283,8 +2299,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, String prefixedKey = Utilities.ReduceField.KEY.toString() + "." + key; ExprNodeDesc end = colExprMap.get(prefixedKey); if (end != null) { - ColStatistics cs = StatsUtils - .getColStatisticsFromExpression(conf, parentStats, end); + ColStatistics cs = StatsUtils.getColStatisticsFromExpression(conf, parentStats, end); if (cs != null) { cs.setColumnName(prefixedKey); colStats.add(cs); @@ -2296,8 +2311,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, String prefixedVal = Utilities.ReduceField.VALUE.toString() + "." + val; ExprNodeDesc end = colExprMap.get(prefixedVal); if (end != null) { - ColStatistics cs = StatsUtils - .getColStatisticsFromExpression(conf, parentStats, end); + ColStatistics cs = StatsUtils.getColStatisticsFromExpression(conf, parentStats, end); if (cs != null) { cs.setColumnName(prefixedVal); colStats.add(cs); @@ -2307,6 +2321,8 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, outStats.setColumnStats(colStats); } + + outStats = applyRuntimeStats(aspCtx.getParseContext().getContext(), outStats, rop, parent.getStatistics()); rop.setStatistics(outStats); if (LOG.isDebugEnabled()) { LOG.debug("[0] STATS-" + rop.toString() + ": " + outStats.extendedToString()); @@ -2355,6 +2371,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, LOG.debug("[0] STATS-" + op.toString() + ": " + stats.extendedToString()); } } + stats = applyRuntimeStats(aspCtx.getParseContext().getContext(), stats, op, stats); op.getConf().setStatistics(stats); } } @@ -2473,4 +2490,28 @@ static boolean satisfyPrecondition(Statistics stats) { return stats != null && stats.getBasicStatsState().equals(Statistics.State.COMPLETE) && !stats.getColumnStatsState().equals(Statistics.State.NONE); } + + private static Statistics applyRuntimeStats(Context context, Statistics stats, Operator op, Statistics maxStat) { + if (!context.getRuntimeStatsSource().isPresent()) { + return stats; + } + RuntimeStatsSource rss = context.getRuntimeStatsSource().get(); + + Optional os = rss.lookup(op); + + if (!os.isPresent()) { + return stats; + } + if (maxStat != null && maxStat.getBasicStatsState() == State.COMPLETE + && os.get().getOutputRecords() > maxStat.getNumRows()) { + return stats; + } + + LOG.debug("using runtime stats for {}; {}", op, os.get()); + Statistics outStats = stats.clone(); + outStats = outStats.scaleToRowCount(os.get().getOutputRecords(), false); + outStats.setRuntimeStats(true); + return outStats; + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g index 78cbf25c43..a1ec96cff9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g @@ -303,6 +303,7 @@ KW_COMPACTIONS: 'COMPACTIONS'; KW_TRANSACTIONS: 'TRANSACTIONS'; KW_REWRITE : 'REWRITE'; KW_AUTHORIZATION: 'AUTHORIZATION'; +KW_REOPTIMIZATION: 'REOPTIMIZATION'; KW_CONF: 'CONF'; KW_VALUES: 'VALUES'; KW_RELOAD: 'RELOAD'; diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 733ec79ce1..88532b1b2f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -773,14 +773,21 @@ explainStatement : KW_EXPLAIN ( explainOption* execStatement -> ^(TOK_EXPLAIN execStatement explainOption*) | - KW_REWRITE queryStatementExpression -> ^(TOK_EXPLAIN_SQ_REWRITE queryStatementExpression)) + KW_REWRITE queryStatementExpression -> ^(TOK_EXPLAIN_SQ_REWRITE queryStatementExpression) + ) ; explainOption @init { msgs.push("explain option"); } @after { msgs.pop(); } - : KW_EXTENDED|KW_FORMATTED|KW_DEPENDENCY|KW_LOGICAL|KW_AUTHORIZATION|KW_ANALYZE| - (KW_VECTORIZATION vectorizationOnly? vectorizatonDetail?) + : KW_EXTENDED + | KW_FORMATTED + | KW_DEPENDENCY + | KW_LOGICAL + | KW_AUTHORIZATION + | KW_ANALYZE + | KW_REOPTIMIZATION + | (KW_VECTORIZATION vectorizationOnly? vectorizatonDetail?) ; vectorizationOnly diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java index 714cf3914b..e04a783892 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java @@ -22,12 +22,9 @@ import java.util.HashMap; import java.util.Map; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.ql.exec.PTFUtils; import org.apache.hadoop.hive.ql.plan.Explain.Level; -import org.apache.hadoop.hive.ql.stats.StatsCollectionContext; -public class AbstractOperatorDesc implements OperatorDesc { +public abstract class AbstractOperatorDesc implements OperatorDesc { protected boolean vectorMode = false; @@ -125,10 +122,12 @@ public void setMaxMemoryAvailable(final long memoryAvailble) { this.memAvailable = memoryAvailble; } + @Override public String getRuntimeStatsTmpDir() { return runtimeStatsTmpDir; } + @Override public void setRuntimeStatsTmpDir(String runtimeStatsTmpDir) { this.runtimeStatsTmpDir = runtimeStatsTmpDir; } @@ -161,4 +160,9 @@ public void setColumnExprMap(Map colExprMap) { this.colExprMap = colExprMap; } + @Override + public void fillSignature(Map ret) { + throw new RuntimeException(); + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/AppMasterEventDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/AppMasterEventDesc.java index 7d5be6ba81..c8fa44627b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/AppMasterEventDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/AppMasterEventDesc.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.plan; import java.io.IOException; -import java.util.List; import java.util.Objects; import org.apache.hadoop.hive.ql.plan.Explain.Level; @@ -27,6 +26,7 @@ import org.apache.hadoop.io.DataOutputBuffer; + @SuppressWarnings("serial") @Explain(displayName = "Application Master Event Operator") public class AppMasterEventDesc extends AbstractOperatorDesc { @@ -36,11 +36,13 @@ private String inputName; @Explain(displayName = "Target Vertex") + @Signature public String getVertexName() { return vertexName; } @Explain(displayName = "Target Input") + @Signature public String getInputName() { return inputName; } @@ -53,6 +55,7 @@ public void setVertexName(String vertexName) { this.vertexName = vertexName; } + @Signature public TableDesc getTable() { return table; } @@ -98,4 +101,5 @@ public boolean isSame(OperatorDesc other) { } return false; } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/CommonMergeJoinDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/CommonMergeJoinDesc.java index 7332693513..8174d4b6ec 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/CommonMergeJoinDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/CommonMergeJoinDesc.java @@ -38,6 +38,7 @@ public CommonMergeJoinDesc(int numBuckets, int mapJoinConversionPos, this.mapJoinConversionPos = mapJoinConversionPos; } + @Signature public int getNumBuckets() { return numBuckets; } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java index 5d3fdb8b63..10f68d3c4c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java @@ -67,6 +67,7 @@ public String displayTargetColumn() { return targetColumnName + " (" + targetColumnType + ")"; } + @Signature public String getTargetColumnName() { return targetColumnName; } @@ -75,6 +76,7 @@ public void setTargetColumnName(String columnName) { this.targetColumnName = columnName; } + @Signature public String getTargetColumnType() { return targetColumnType; } @@ -94,6 +96,7 @@ public void setPartKey(ExprNodeDesc partKey) { } @Explain(displayName = "Partition key expr") + @Signature public String getPartKeyString() { return this.partKey.getExprString(); } @@ -112,4 +115,5 @@ public boolean isSame(OperatorDesc other) { } return false; } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java index ce61fc5a2e..d83cbdf223 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java @@ -191,6 +191,7 @@ public void setIsUsingThriftJDBCBinarySerDe(boolean isUsingThriftJDBCBinarySerDe } @Explain(displayName = "directory", explainLevels = { Level.EXTENDED }) + @Signature public Path getDirName() { return dirName; } @@ -214,6 +215,7 @@ public Path getMergeInputDirName() { } @Explain(displayName = "table", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Signature public TableDesc getTableInfo() { return tableInfo; } @@ -223,6 +225,7 @@ public void setTableInfo(final TableDesc tableInfo) { } @Explain(displayName = "compressed") + @Signature public boolean getCompressed() { return compressed; } @@ -232,6 +235,8 @@ public void setCompressed(boolean compressed) { } @Explain(displayName = "GlobalTableId", explainLevels = { Level.EXTENDED }) + @Signature + public int getDestTableId() { return destTableId; } @@ -260,6 +265,8 @@ public void setCompressType(String intermediateCompressType) { * @return the multiFileSpray */ @Explain(displayName = "MultiFileSpray", explainLevels = { Level.EXTENDED }) + @Signature + public boolean isMultiFileSpray() { return multiFileSpray; } @@ -311,6 +318,8 @@ public void setCanBeMerged(boolean canBeMerged) { * @return the totalFiles */ @Explain(displayName = "TotalFiles", explainLevels = { Level.EXTENDED }) + @Signature + public int getTotalFiles() { return totalFiles; } @@ -340,6 +349,8 @@ public void setPartitionCols(ArrayList partitionCols) { * @return the numFiles */ @Explain(displayName = "NumFilesPerFileSink", explainLevels = { Level.EXTENDED }) + @Signature + public int getNumFiles() { return numFiles; } @@ -364,6 +375,7 @@ public void setStaticSpec(String staticSpec) { } @Explain(displayName = "Static Partition Specification", explainLevels = { Level.EXTENDED }) + @Signature public String getStaticSpec() { return staticSpec; } @@ -374,6 +386,8 @@ public void setGatherStats(boolean gatherStats) { @Override @Explain(displayName = "GatherStats", explainLevels = { Level.EXTENDED }) + @Signature + public boolean isGatherStats() { return gatherStats; } @@ -391,6 +405,9 @@ public boolean isGatherStats() { */ @Override @Explain(displayName = "Stats Publishing Key Prefix", explainLevels = { Level.EXTENDED }) + // FIXME: including this in the signature will almost certenly differ even if the operator is doing the same + // there might be conflicting usages of logicalCompare? + @Signature public String getStatsAggPrefix() { // dirName uniquely identifies destination directory of a FileSinkOperator. // If more than one FileSinkOperator write to the same partition, this dirName diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java index d59834ce08..3ef30ae193 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java @@ -109,6 +109,7 @@ public FilterDesc( this.sampleDescr = sampleDescr; } + @Signature public String getPredicateString() { return PlanUtils.getExprListString(Arrays.asList(predicate)); } @@ -137,6 +138,7 @@ public void setPredicate( } @Explain(displayName = "isSamplingPred", explainLevels = { Level.EXTENDED }) + @Signature public boolean getIsSamplingPred() { return isSamplingPred; } @@ -154,6 +156,7 @@ public void setSampleDescr(final SampleDesc sampleDescr) { } @Explain(displayName = "sampleDesc", explainLevels = { Level.EXTENDED }) + @Signature public String getSampleDescExpr() { return sampleDescr == null ? null : sampleDescr.toString(); } @@ -234,4 +237,5 @@ public boolean isSame(OperatorDesc other) { } return false; } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java index 86cc77d43b..662de1c31d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java @@ -25,16 +25,11 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationDesc; -import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression; import org.apache.hadoop.hive.ql.udf.UDFType; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; import org.apache.hive.common.util.AnnotationUtils; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.optimizer.physical.Vectorizer; import org.apache.hadoop.hive.ql.plan.Explain.Level; import org.apache.hadoop.hive.ql.plan.Explain.Vectorization; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; /** @@ -129,6 +124,7 @@ public Mode getMode() { } @Explain(displayName = "mode") + @Signature public String getModeString() { switch (mode) { case COMPLETE: @@ -155,6 +151,7 @@ public void setMode(final Mode mode) { } @Explain(displayName = "keys") + @Signature public String getKeyString() { return PlanUtils.getExprListString(keys); } @@ -173,6 +170,7 @@ public void setKeys(final ArrayList keys) { } @Explain(displayName = "outputColumnNames") + @Signature public ArrayList getOutputColumnNames() { return outputColumnNames; } @@ -183,6 +181,7 @@ public void setKeys(final ArrayList keys) { } @Explain(displayName = "pruneGroupingSetId", displayOnlyOnTrue = true) + @Signature public boolean pruneGroupingSetId() { return groupingSetPosition >= 0 && outputColumnNames.size() != keys.size() + aggregators.size(); @@ -210,6 +209,7 @@ public void setMemoryThreshold(float memoryThreshold) { } @Explain(displayName = "aggregations", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Signature public List getAggregatorStrings() { List res = new ArrayList(); for (AggregationDesc agg: aggregators) { @@ -235,6 +235,7 @@ public boolean isAggregate() { } @Explain(displayName = "bucketGroup", displayOnlyOnTrue = true) + @Signature public boolean getBucketGroup() { return bucketGroup; } @@ -424,4 +425,5 @@ public boolean isSame(OperatorDesc other) { return false; } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java index 9c651ab3ab..e6a177a226 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java @@ -289,6 +289,7 @@ public void setFilterMap(int[][] filterMap) { @Override @Explain(displayName = "filter mappings", explainLevels = { Level.EXTENDED }) + @Signature public Map getFilterMapString() { return toCompactString(filterMap); } @@ -304,6 +305,7 @@ public void setRetainList(Map> retainList) { /** * @return the keys in string form */ + @Override @Explain(displayName = "keys") public Map getKeysString() { Map keyMap = new LinkedHashMap(); @@ -313,6 +315,7 @@ public void setRetainList(Map> retainList) { return keyMap; } + @Override @Explain(displayName = "keys", explainLevels = { Level.USER }) public Map getUserLevelExplainKeysString() { Map keyMap = new LinkedHashMap(); @@ -399,4 +402,5 @@ public boolean isSame(OperatorDesc other) { } return false; } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/JoinCondDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/JoinCondDesc.java index 6dcf05af28..ea22131c56 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/JoinCondDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/JoinCondDesc.java @@ -30,7 +30,7 @@ /** * Join conditions Descriptor implementation. - * + * */ public class JoinCondDesc implements Serializable { private static final long serialVersionUID = 1L; @@ -153,7 +153,7 @@ public String getJoinCondString() { @Explain(explainLevels = { Level.USER }) public String getUserLevelJoinCondString() { - JSONObject join = new JSONObject(new LinkedHashMap()); + JSONObject join = new JSONObject(new LinkedHashMap<>()); try { switch (type) { case JoinDesc.INNER_JOIN: @@ -200,4 +200,6 @@ public boolean equals(Object obj) { } return true; } + + // XXX: is hashCode missing here? } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java index bd45c752e1..48d50f95d2 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java @@ -229,6 +229,7 @@ public void setReversedExprs(Map reversedExprs) { * @return the keys in string form */ @Explain(displayName = "keys") + @Signature public Map getKeysString() { if (joinKeys == null) { return null; @@ -266,6 +267,7 @@ public void setExprs(final Map> exprs) { * @return Map from alias to filters on the alias. */ @Explain(displayName = "filter predicates") + @Signature public Map getFiltersStringMap() { if (getFilters() == null || getFilters().size() == 0) { return null; @@ -342,6 +344,7 @@ public void setResidualFilterExprs(List residualFilterExprs) { } @Explain(displayName = "outputColumnNames") + @Signature public List getOutputColumnNames() { return outputColumnNames; } @@ -365,6 +368,7 @@ public void setNoOuterJoin(final boolean noOuterJoin) { } @Explain(displayName = "condition map", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Signature public List getCondsList() { if (conds == null) { return null; @@ -425,6 +429,7 @@ public void setTagOrder(Byte[] tagOrder) { } @Explain(displayName = "handleSkewJoin", displayOnlyOnTrue = true) + @Signature public boolean getHandleSkewJoin() { return handleSkewJoin; } @@ -524,6 +529,7 @@ public void setNullSafes(boolean[] nullSafes) { } @Explain(displayName = "nullSafes") + @Signature public String getNullSafeString() { if (nullsafes == null) { return null; @@ -740,4 +746,5 @@ public boolean isSame(OperatorDesc other) { } return false; } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/LateralViewJoinDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/LateralViewJoinDesc.java index 3837a49934..4496dc46c6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/LateralViewJoinDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/LateralViewJoinDesc.java @@ -49,6 +49,7 @@ public void setOutputInternalColNames(ArrayList outputInternalColNames) } @Explain(displayName = "outputColumnNames") + @Signature public ArrayList getOutputInternalColNames() { return outputInternalColNames; } @@ -74,4 +75,5 @@ public boolean isSame(OperatorDesc other) { } return false; } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/LimitDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/LimitDesc.java index ce53feae00..9e52c6c86e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/LimitDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/LimitDesc.java @@ -58,6 +58,7 @@ public void setOffset(Integer offset) { this.offset = offset; } + @Signature @Explain(displayName = "Number of rows", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) public int getLimit() { return limit; @@ -100,4 +101,5 @@ public boolean isSame(OperatorDesc other) { } return false; } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java index cf4ab606f2..b1d9d79f99 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java @@ -139,6 +139,7 @@ private void initRetainExprList() { } @Explain(displayName = "input vertices", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Signature public Map getParentToInput() { return parentToInput; } @@ -156,6 +157,7 @@ public void setParentToInput(Map parentToInput) { } @Explain(displayName = "Estimated key counts", explainLevels = { Level.EXTENDED }) + @Signature public String getKeyCountsExplainDesc() { StringBuilder result = null; for (Map.Entry entry : parentKeyCounts.entrySet()) { @@ -250,6 +252,7 @@ public void setKeys(Map> keys) { * @return the position of the big table not in memory */ @Explain(displayName = "Position of Big Table", explainLevels = { Level.EXTENDED }) + @Signature public int getPosBigTable() { return posBigTable; } @@ -340,6 +343,7 @@ public void setBigTablePartSpecToFileMapping(Map> partToFil } @Explain(displayName = "BucketMapJoin", explainLevels = { Level.USER, Level.EXTENDED }, displayOnlyOnTrue = true) + @Signature public boolean isBucketMapJoin() { return isBucketMapJoin; } @@ -607,4 +611,5 @@ public boolean isSame(OperatorDesc other) { } return false; } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java index 870b4d9a26..e8a5827d7f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java @@ -34,7 +34,10 @@ public void setMaxMemoryAvailable(long memoryAvailble); public String getRuntimeStatsTmpDir(); public void setRuntimeStatsTmpDir(String runtimeStatsTmpDir); + boolean isSame(OperatorDesc other); public Map getColumnExprMap(); public void setColumnExprMap(Map colExprMap); + + void fillSignature(Map ret); } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java index bf24ff8f09..9e166ce018 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java @@ -97,7 +97,7 @@ private float topNMemoryUsage = -1; private boolean mapGroupBy; // for group-by, values with same key on top-K should be forwarded //flag used to control how TopN handled for PTF/Windowing partitions. - private boolean isPTFReduceSink = false; + private boolean isPTFReduceSink = false; private boolean skipTag; // Skip writing tags when feeding into mapjoin hashtable private boolean forwarding; // Whether this RS can forward records directly instead of shuffling/sorting @@ -206,6 +206,7 @@ public String getKeyColString() { return PlanUtils.getExprListString(keyCols); } + @Signature public java.util.ArrayList getKeyCols() { return keyCols; } @@ -227,6 +228,7 @@ public String getValueColsString() { return PlanUtils.getExprListString(valueCols); } + @Signature public java.util.ArrayList getValueCols() { return valueCols; } @@ -245,6 +247,7 @@ public String getUserLevelExplainParitionColsString() { return PlanUtils.getExprListString(partitionCols, true); } + @Signature public java.util.ArrayList getPartitionCols() { return partitionCols; } @@ -261,6 +264,7 @@ public boolean isPartitioning() { return false; } + @Signature @Explain(displayName = "tag", explainLevels = { Level.EXTENDED }) public int getTag() { return tag; @@ -270,6 +274,7 @@ public void setTag(int tag) { this.tag = tag; } + @Signature public int getTopN() { return topN; } @@ -349,6 +354,7 @@ public void setValueSerializeInfo(TableDesc valueSerializeInfo) { * of the same length as key columns, that consists of only "+" * (ascending order) and "-" (descending order). */ + @Signature @Explain(displayName = "sort order") public String getOrder() { return keySerializeInfo.getProperties().getProperty( @@ -437,6 +443,7 @@ public boolean isForwarding() { return forwarding; } + @Signature @Explain(displayName = "auto parallelism", explainLevels = { Level.EXTENDED }) public final boolean isAutoParallel() { return (this.reduceTraits.contains(ReducerTraits.AUTOPARALLEL)); @@ -462,7 +469,7 @@ public final void setReducerTraits(EnumSet traits) { // reducers or hash function. boolean wasUnset = this.reduceTraits.remove(ReducerTraits.UNSET); - + if (this.reduceTraits.contains(ReducerTraits.FIXED)) { return; } else if (traits.contains(ReducerTraits.FIXED)) { @@ -661,4 +668,5 @@ public boolean isSame(OperatorDesc other) { } return false; } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ScriptDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/ScriptDesc.java index 858de98229..b0d814c9e9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ScriptDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ScriptDesc.java @@ -18,12 +18,12 @@ package org.apache.hadoop.hive.ql.plan; +import java.util.Objects; + import org.apache.hadoop.hive.ql.exec.RecordReader; import org.apache.hadoop.hive.ql.exec.RecordWriter; import org.apache.hadoop.hive.ql.plan.Explain.Level; -import java.util.Objects; - /** * ScriptDesc. @@ -63,6 +63,7 @@ public ScriptDesc(final String scriptCmd, final TableDesc scriptInputInfo, this.scriptErrInfo = scriptErrInfo; } + @Signature @Explain(displayName = "command", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) public String getScriptCmd() { return scriptCmd; @@ -72,6 +73,7 @@ public void setScriptCmd(final String scriptCmd) { this.scriptCmd = scriptCmd; } + @Signature @Explain(displayName = "output info", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) public TableDesc getScriptOutputInfo() { return scriptOutputInfo; @@ -154,4 +156,5 @@ public boolean isSame(OperatorDesc other) { } return false; } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java index e38e7e468f..d28a9d98e8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java @@ -72,6 +72,7 @@ public Object clone() { return ret; } + @Signature @Explain(displayName = "expressions") public String getColListString() { return PlanUtils.getExprListString(colList); @@ -86,6 +87,7 @@ public void setColList( this.colList = colList; } + @Signature @Explain(displayName = "outputColumnNames") public List getOutputColumnNames() { return outputColumnNames; @@ -101,6 +103,7 @@ public void setOutputColumnNames( this.outputColumnNames = outputColumnNames; } + @Signature @Explain(displayName = "SELECT * ") public String explainNoCompute() { if (isSelStarNoCompute()) { @@ -184,4 +187,5 @@ public boolean isSame(OperatorDesc other) { } return false; } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/Signature.java ql/src/java/org/apache/hadoop/hive/ql/plan/Signature.java new file mode 100644 index 0000000000..a2c3a0dc3c --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/plan/Signature.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.plan; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(value = ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +public @interface Signature { + + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/SignatureUtils1.java ql/src/java/org/apache/hadoop/hive/ql/plan/SignatureUtils1.java new file mode 100644 index 0000000000..b9727647b6 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/plan/SignatureUtils1.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.plan; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.google.common.collect.Sets; + +//FIXME: rename? +//FIXME: package? +//FIXME: apidoc +public class SignatureUtils1 { + + private static Map, SignatureMapper> mappers = new HashMap<>(); + + public static void write(Map ret, Object o) { + SignatureMapper mapper = getSigMapper(o.getClass()); + mapper.write(ret, o); + } + + static class SignatureMapper { + + static final Set acceptedSignatureTypes = Sets.newHashSet(); + + private List sigMethods; + + public SignatureMapper(Class o) { + Method[] f = o.getMethods(); + sigMethods = new ArrayList<>(); + for (Method method : f) { + if (method.isAnnotationPresent(Signature.class)) { + Class rType = method.getReturnType(); + String rTypeName = rType.getName(); + if (!rType.isPrimitive() && acceptedSignatureTypes.contains(rTypeName)) { + throw new RuntimeException("unxepected type (" + rTypeName + ") used in signature"); + } + sigMethods.add(method); + } + } + } + + public void write(Map ret, Object o) { + if (sigMethods.isEmpty()) { + // by supplying using "o" this enforces identity/equls matching + // which will most probably make the signature very unique + ret.put(o.getClass().getName(), o); + } else { + ret.put(o.getClass().getName(), "1"); + for (Method method : sigMethods) { + try { + Object res = method.invoke(o); + ret.put(method.getName(), res); + } catch (Exception e) { + throw new RuntimeException("Error invoking signature method", e); + } + } + } + } + + } + + private static SignatureMapper getSigMapper(Class o) { + SignatureMapper m = mappers.get(o); + if (m == null) { + m = new SignatureMapper(o); + mappers.put(o, m); + } + return m; + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java index 0057f0c2c6..fd461ae529 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java @@ -49,6 +49,7 @@ boolean morePreciseThan(State other) { private State basicStatsState; private Map columnStats; private State columnStatsState; + private boolean runtimeStats; public Statistics() { this(0, 0); @@ -119,6 +120,9 @@ public void setColumnStatsState(State columnStatsState) { @Explain(displayName = "Statistics") public String toString() { StringBuilder sb = new StringBuilder(); + if (runtimeStats) { + sb.append("(RUNTIME) "); + } sb.append("Num rows: "); sb.append(numRows); if (runTimeNumRows >= 0) { @@ -136,6 +140,9 @@ public String toString() { @Explain(displayName = "Statistics", explainLevels = { Level.USER }) public String toUserLevelExplainString() { StringBuilder sb = new StringBuilder(); + if (runtimeStats) { + sb.append("runtime: "); + } sb.append("rows="); sb.append(numRows); if (runTimeNumRows >= 0) { @@ -153,6 +160,9 @@ public String toUserLevelExplainString() { public String extendedToString() { StringBuilder sb = new StringBuilder(); + if (runtimeStats) { + sb.append(" (runtime) "); + } sb.append(" numRows: "); sb.append(numRows); sb.append(" dataSize: "); @@ -179,6 +189,8 @@ public Statistics clone() { } clone.setColumnStats(cloneColStats); } + // TODO: this boolean flag is set only by RS stats annotation at this point + //clone.setRuntimeStats(runtimeStats); return clone; } @@ -300,10 +312,13 @@ public void setRunTimeNumRows(long runTimeNumRows) { this.runTimeNumRows = runTimeNumRows; } - public Statistics scaleToRowCount(long newRowCount) { + public Statistics scaleToRowCount(long newRowCount, boolean downScaleOnly) { Statistics ret; ret = clone(); - if(numRows == 0 || newRowCount >= numRows) { + if (numRows == 0) { + return ret; + } + if (downScaleOnly && newRowCount >= numRows) { return ret; } // FIXME: using real scaling by new/old ration might yield better results? @@ -311,4 +326,12 @@ public Statistics scaleToRowCount(long newRowCount) { ret.dataSize = StatsUtils.safeMult(getAvgRowSize(), newRowCount); return ret; } + + public boolean isRuntimeStats() { + return runtimeStats; + } + + public void setRuntimeStats(final boolean runtimeStats) { + this.runtimeStats = runtimeStats; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java index 59968fa8ef..9d0df704eb 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java @@ -156,10 +156,20 @@ public Object clone() { } @Explain(displayName = "alias") + // FIXME: this might not needed to be in the signature; but in that case the compare shouldn't consider it either! + @Signature public String getAlias() { return alias; } + @Signature + public String getPredicateString() { + if (filterExpr == null) { + return null; + } + return PlanUtils.getExprListString(Arrays.asList(filterExpr)); + } + @Explain(displayName = "table", jsonOnly = true) public String getTableName() { return this.tableName; @@ -219,6 +229,7 @@ public String getFilterExprString() { return PlanUtils.getExprListString(Arrays.asList(filterExpr)); } + // @Signature // XXX public ExprNodeGenericFuncDesc getFilterExpr() { return filterExpr; } @@ -296,6 +307,7 @@ public void setGatherStats(boolean gatherStats) { @Override @Explain(displayName = "GatherStats", explainLevels = { Level.EXTENDED }) + @Signature public boolean isGatherStats() { return gatherStats; } @@ -347,6 +359,7 @@ public void setRowLimit(int rowLimit) { this.rowLimit = rowLimit; } + @Signature public int getRowLimit() { return rowLimit; } @@ -372,6 +385,11 @@ public boolean getIsMetadataOnly() { return isMetadataOnly; } + // @Signature + public String getQualifiedTable() { + return tableMetadata.getFullyQualifiedName(); + } + public Table getTableMetadata() { return tableMetadata; } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/UDTFDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/UDTFDesc.java index cf8e6e547e..6193fe4234 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/UDTFDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/UDTFDesc.java @@ -19,10 +19,11 @@ package org.apache.hadoop.hive.ql.plan; import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF; -import org.apache.hadoop.hive.ql.plan.Explain.Level; import java.util.Objects; +import org.apache.hadoop.hive.ql.plan.Explain.Level; + /** * All member variables should have a setters and getters of the form get class1) { + return false; + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/GroupTransformer.java ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/GroupTransformer.java new file mode 100644 index 0000000000..243c817aeb --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/GroupTransformer.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.plan.mapper; + +import org.apache.hadoop.hive.ql.plan.mapper.PlanMapper.LinkGroup; + +// FIXME: seems more like a visitor? +public interface GroupTransformer { + void map(LinkGroup group); +} \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/PlanMapper.java ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/PlanMapper.java new file mode 100644 index 0000000000..a7ca084c48 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/PlanMapper.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.plan.mapper; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Set; + +import com.google.common.annotations.VisibleForTesting; + +/** + * Enables to connect related objects to eachother. + * + * Most importantly it aids to connect Operators to OperatorStats and probably RelNodes. + */ +public class PlanMapper { + + Set groups = new HashSet<>(); + private Map objectMap = new HashMap<>(); + + public class LinkGroup { + Set members = new HashSet<>(); + + public void add(Object o) { + members.add(o); + objectMap.put(o, this); + } + + @SuppressWarnings("unchecked") + public List getAll(Class clazz) { + List ret = new ArrayList<>(); + for (Object m : members) { + if (clazz.isInstance(m)) { + ret.add((T) m); + } + } + return ret; + } + } + + public void link(Object o1, Object o2) { + LinkGroup g1 = objectMap.get(o1); + LinkGroup g2 = objectMap.get(o2); + if (g1 != null && g2 != null && g1 != g2) { + throw new RuntimeException("equivalence mapping violation"); + } + LinkGroup targetGroup = (g1 != null) ? g1 : (g2 != null ? g2 : new LinkGroup()); + groups.add(targetGroup); + targetGroup.add(o1); + targetGroup.add(o2); + } + + public List getAll(Class clazz) { + List ret = new ArrayList<>(); + for (LinkGroup g : groups) { + ret.addAll(g.getAll(clazz)); + } + return ret; + } + + public void runMapper(GroupTransformer mapper) { + for (LinkGroup equivGroup : groups) { + mapper.map(equivGroup); + } + } + + public List lookupAll(Class clazz, Object key) { + LinkGroup group = objectMap.get(key); + if (group == null) { + throw new NoSuchElementException(Objects.toString(key)); + } + return group.getAll(clazz); + } + + public T lookup(Class clazz, Object key) { + List all = lookupAll(clazz, key); + if (all.size() != 1) { + // FIXME: use a different exception type? + throw new IllegalArgumentException("Expected match count is 1; but got:" + all); + } + return all.get(0); + } + + // FIXME: find a more natural way... + @VisibleForTesting + public Iterator iterateGroups() { + return groups.iterator(); + + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/PlanMapperProcess.java ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/PlanMapperProcess.java new file mode 100644 index 0000000000..e3c388181e --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/PlanMapperProcess.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.plan.mapper; + +import java.util.List; + +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.optimizer.signature.OpTreeSignature; +import org.apache.hadoop.hive.ql.optimizer.signature.OpTreeSignatureFactory; +import org.apache.hadoop.hive.ql.plan.mapper.PlanMapper.LinkGroup; + +public class PlanMapperProcess { + + private static class OpTreeSignatureMapper implements GroupTransformer { + + private OpTreeSignatureFactory cache = OpTreeSignatureFactory.newCache(); + + @Override + public void map(LinkGroup group) { + List filters = group.getAll(Operator.class); + for (Operator op : filters) { + group.add(OpTreeSignature.of(op,cache)); + } + } + } + + public static void runPostProcess(PlanMapper planMapper) { + planMapper.runMapper(new OpTreeSignatureMapper()); + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/RuntimeStatsSource.java ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/RuntimeStatsSource.java new file mode 100644 index 0000000000..21a0678153 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/RuntimeStatsSource.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.plan.mapper; + +import java.util.Optional; + +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.stats.OperatorStats; + +public interface RuntimeStatsSource extends StatsSource { + public Optional lookup(Operator tsop); + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/SimpleRuntimeStatsSource.java ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/SimpleRuntimeStatsSource.java new file mode 100644 index 0000000000..6f340b8450 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/SimpleRuntimeStatsSource.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.plan.mapper; + +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Optional; + +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; +import org.apache.hadoop.hive.ql.optimizer.signature.OpTreeSignature; +import org.apache.hadoop.hive.ql.stats.OperatorStats; + +public class SimpleRuntimeStatsSource implements RuntimeStatsSource { + + private final PlanMapper pm; + + + public SimpleRuntimeStatsSource(PlanMapper pm) { + PlanMapperProcess.runPostProcess(pm); + this.pm = pm; + } + + @Override + public Optional lookup(Operator op) { + try { + OpTreeSignature sig = OpTreeSignature.of(op); + List v = pm.lookupAll(OperatorStats.class, sig); + if (v.size() > 0) { + return Optional.of(v.get(0)); + } + return Optional.empty(); + } catch (NoSuchElementException | IllegalArgumentException iae) { + return Optional.empty(); + } + } + + @Override + public boolean canProvideStatsFor(Class class1) { + if (Operator.class.isAssignableFrom(class1)) { + return true; + } + if (HiveFilter.class.isAssignableFrom(class1)) { + return true; + } + return false; + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/StatsSource.java ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/StatsSource.java new file mode 100644 index 0000000000..a4cb6e9771 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/StatsSource.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.plan.mapper; + +public interface StatsSource { + + boolean canProvideStatsFor(Class class1); + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/reexec/IReExecutionPlugin.java ql/src/java/org/apache/hadoop/hive/ql/reexec/IReExecutionPlugin.java new file mode 100644 index 0000000000..9a11707594 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/reexec/IReExecutionPlugin.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.reexec; + +import org.apache.hadoop.hive.common.classification.InterfaceAudience; +import org.apache.hadoop.hive.common.classification.InterfaceStability; +import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.plan.mapper.PlanMapper; + +/** + * Defines an interface for re-execution logics. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public interface IReExecutionPlugin { + + /** + * Called when the {@link Driver} is being initialized + * + * The plugin may add hooks/etc to tap into the system. + */ + void initialize(Driver driver); + + /** + * The query have failed, does this plugin advises to re-execute it again? + */ + boolean shouldReExecute(int executionNum); + + /** + * The plugin should prepare for the re-compilaton of the query. + */ + void prepareToReExecute2(); + + /** + * The query have failed; and have been recompiled - does this plugin advises to re-execute it again? + */ + boolean shouldReExecute2(int executionNum, PlanMapper oldPlanMapper, PlanMapper newPlanMapper); + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecDriver.java ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecDriver.java new file mode 100644 index 0000000000..c847c099bc --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecDriver.java @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.reexec; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import org.antlr.runtime.tree.Tree; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.api.Schema; +import org.apache.hadoop.hive.ql.Context; +import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.IDriver; +import org.apache.hadoop.hive.ql.QueryDisplay; +import org.apache.hadoop.hive.ql.QueryInfo; +import org.apache.hadoop.hive.ql.QueryPlan; +import org.apache.hadoop.hive.ql.QueryState; +import org.apache.hadoop.hive.ql.exec.FetchTask; +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.HiveParser; +import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHook; +import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHookContext; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.mapper.PlanMapper; +import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Enables to use re-execution logics. + * + * Covers the IDriver interface, handles query re-execution; and asks clear questions from the underlying re-execution plugins. + */ +public class ReExecDriver implements IDriver { + + private class HandleReOptimizationExplain implements HiveSemanticAnalyzerHook { + + @Override + public ASTNode preAnalyze(HiveSemanticAnalyzerHookContext context, ASTNode ast) throws SemanticException { + if (ast.getType() == HiveParser.TOK_EXPLAIN) { + int childCount = ast.getChildCount(); + for (int i = 1; i < childCount; i++) { + if (ast.getChild(i).getType() == HiveParser.KW_REOPTIMIZATION) { + explainReOptimization = true; + ast.deleteChild(i); + break; + } + } + if (explainReOptimization && firstExecution()) { + Tree execTree = ast.getChild(0); + execTree.setParent(ast.getParent()); + ast.getParent().setChild(0, execTree); + return (ASTNode) execTree; + } + } + return ast; + } + + @Override + public void postAnalyze(HiveSemanticAnalyzerHookContext context, List> rootTasks) + throws SemanticException { + } + } + + private static final Logger LOG = LoggerFactory.getLogger(ReExecDriver.class); + private boolean explainReOptimization; + protected Driver coreDriver; + private QueryState queryState; + private String currentQuery; + private int executionIndex; + + private ArrayList plugins; + + @Override + public HiveConf getConf() { + return queryState.getConf(); + } + + public boolean firstExecution() { + return executionIndex == 0; + } + + public ReExecDriver(QueryState queryState, String userName, QueryInfo queryInfo, + ArrayList plugins) { + this.queryState = queryState; + coreDriver = new Driver(queryState, userName, queryInfo, null); + coreDriver.getHookRunner().addSemanticAnalyzerHook(new HandleReOptimizationExplain()); + this.plugins = plugins; + + for (IReExecutionPlugin p : plugins) { + p.initialize(coreDriver); + } + } + + @Override + public int compile(String string) { + return coreDriver.compile(string); + } + + @Override + public CommandProcessorResponse compileAndRespond(String statement) { + currentQuery = statement; + return coreDriver.compileAndRespond(statement); + } + + @Override + public QueryPlan getPlan() { + return coreDriver.getPlan(); + } + + @Override + public QueryDisplay getQueryDisplay() { + return coreDriver.getQueryDisplay(); + } + + @Override + public void setOperationId(String guid64) { + coreDriver.setOperationId(guid64); + } + + @Override + public CommandProcessorResponse run() { + executionIndex = 0; + int maxExecutuions = 1 + coreDriver.getConf().getIntVar(ConfVars.HIVE_QUERY_MAX_REEXECUTION_COUNT); + + while (true) { + executionIndex++; + coreDriver.getContext().setExecutionIndex(executionIndex); + LOG.info("Execution #{} of query", executionIndex); + CommandProcessorResponse cpr = coreDriver.run(); + + boolean shouldReExecute = explainReOptimization && executionIndex==1; + shouldReExecute |= cpr.getResponseCode() != 0 && shouldReExecute(); + + if (executionIndex >= maxExecutuions || !shouldReExecute) { + return cpr; + } + LOG.info("Preparing to re-execute query"); + prepareToReExecute(); + PlanMapper oldPlanMapper = coreDriver.getPlanMapper(); + CommandProcessorResponse compile_resp = coreDriver.compileAndRespond(currentQuery); + if (compile_resp.failed()) { + // FIXME: somehow place pointers that re-execution compilation have failed; the query have been successfully compiled before? + return compile_resp; + } + + PlanMapper newPlanMapper = coreDriver.getPlanMapper(); + if (!explainReOptimization && !shouldReExecuteAfterCompile(oldPlanMapper, newPlanMapper)) { + // FIXME: retain old error; or create a new one? + return cpr; + } + } + } + + private boolean shouldReExecuteAfterCompile(PlanMapper oldPlanMapper, PlanMapper newPlanMapper) { + boolean ret = false; + for (IReExecutionPlugin p : plugins) { + ret |= p.shouldReExecute2(executionIndex, oldPlanMapper, newPlanMapper); + } + return ret; + } + + private boolean shouldReExecute() { + boolean ret = false; + for (IReExecutionPlugin p : plugins) { + ret |= p.shouldReExecute(executionIndex); + } + return ret; + } + + @Override + public CommandProcessorResponse run(String command) { + CommandProcessorResponse r0 = compileAndRespond(command); + if (r0.getResponseCode() != 0) { + return r0; + } + return run(); + } + + protected void prepareToReExecute() { + for (IReExecutionPlugin p : plugins) { + p.prepareToReExecute2(); + } + } + + @Override + public boolean getResults(List res) throws IOException { + return coreDriver.getResults(res); + } + + @Override + public void setMaxRows(int maxRows) { + coreDriver.setMaxRows(maxRows); + } + + @Override + public FetchTask getFetchTask() { + return coreDriver.getFetchTask(); + } + + @Override + public Schema getSchema() { + return coreDriver.getSchema(); + } + + @Override + public boolean isFetchingTable() { + return coreDriver.isFetchingTable(); + } + + @Override + public void resetFetch() throws IOException { + coreDriver.resetFetch(); + } + + @Override + public void close() { + coreDriver.close(); + } + + @Override + public void destroy() { + coreDriver.destroy(); + } + + @Override + public final Context getContext() { + return coreDriver.getContext(); + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecutionOverlayPlugin.java ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecutionOverlayPlugin.java new file mode 100644 index 0000000000..a1c711831e --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecutionOverlayPlugin.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.reexec; + +import java.util.Map; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext; +import org.apache.hadoop.hive.ql.hooks.HookContext; +import org.apache.hadoop.hive.ql.hooks.HookContext.HookType; +import org.apache.hadoop.hive.ql.plan.mapper.PlanMapper; + +/** + * Re-Executes a query only adding an extra overlay + */ +public class ReExecutionOverlayPlugin implements IReExecutionPlugin { + + private Driver driver; + private Map subtree; + + class LocalHook implements ExecuteWithHookContext { + + @Override + public void run(HookContext hookContext) throws Exception { + if (hookContext.getHookType() == HookType.ON_FAILURE_HOOK) { + Throwable exception = hookContext.getException(); + if (exception != null) { + if (exception.getMessage().contains("Vertex failed,")) { + retryPossible = true; + } + } + } + } + } + + @Override + public void initialize(Driver driver) { + this.driver = driver; + driver.getHookRunner().addOnFailureHook(new LocalHook()); + HiveConf conf = driver.getConf(); + subtree = conf.subtree("reexec.overlay"); + } + + private boolean retryPossible; + + @Override + public void prepareToReExecute2() { + HiveConf conf = driver.getConf(); + conf.verifyAndSetAll(subtree); + } + + @Override + public boolean shouldReExecute(int executionNum) { + return executionNum == 1 && !subtree.isEmpty() && retryPossible; + } + + @Override + public boolean shouldReExecute2(int executionNum, PlanMapper pm1, PlanMapper pm2) { + return executionNum == 1; + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/reexec/ReOptimizePlugin.java ql/src/java/org/apache/hadoop/hive/ql/reexec/ReOptimizePlugin.java new file mode 100644 index 0000000000..9796773e31 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/reexec/ReOptimizePlugin.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.reexec; + +import java.util.Iterator; +import java.util.List; + +import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.mapjoin.MapJoinMemoryExhaustionError; +import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext; +import org.apache.hadoop.hive.ql.hooks.HookContext; +import org.apache.hadoop.hive.ql.hooks.HookContext.HookType; +import org.apache.hadoop.hive.ql.plan.mapper.PlanMapper; +import org.apache.hadoop.hive.ql.plan.mapper.SimpleRuntimeStatsSource; +import org.apache.hadoop.hive.ql.stats.OperatorStatsReaderHook; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + */ +public class ReOptimizePlugin implements IReExecutionPlugin { + + private static final Logger LOG = LoggerFactory.getLogger(ReOptimizePlugin.class); + + private boolean retryPossible; + + private Driver coreDriver; + + class LocalHook implements ExecuteWithHookContext { + + @Override + public void run(HookContext hookContext) throws Exception { + if (hookContext.getHookType() == HookType.ON_FAILURE_HOOK) { + Throwable exception = hookContext.getException(); + if (exception != null) { + { + String message = exception.getMessage(); + if (message != null) { + boolean isOOM = message.contains(MapJoinMemoryExhaustionError.class.getName()) + || message.contains(OutOfMemoryError.class.getName()); + if (message.contains("Vertex failed,") && isOOM) { + retryPossible = true; + } + System.out.println(exception); + } + } + } + } + } + } + + @Override + public void initialize(Driver driver) { + coreDriver = driver; + coreDriver.getHookRunner().addOnFailureHook(new LocalHook()); + OperatorStatsReaderHook hook = new OperatorStatsReaderHook(); + coreDriver.getHookRunner().addOnFailureHook(hook); + coreDriver.getHookRunner().addPostHook(hook); + } + + @Override + public boolean shouldReExecute(int executionNum) { + return retryPossible; + } + + @Override + public void prepareToReExecute2() { + PlanMapper pm = coreDriver.getContext().getPlanMapper(); + coreDriver.setRuntimeStatsSource(new SimpleRuntimeStatsSource(pm)); + retryPossible = false; + } + + @Override + public boolean shouldReExecute2(int executionNum, PlanMapper oldPlanMapper, PlanMapper newPlanMapper) { + return planDidChange(oldPlanMapper, newPlanMapper); + } + + private boolean planDidChange(PlanMapper pmL, PlanMapper pmR) { + List opsL = getRootOps(pmL); + List opsR = getRootOps(pmR); + for (Iterator itL = opsL.iterator(); itL.hasNext();) { + Operator opL = itL.next(); + for (Iterator itR = opsR.iterator(); itR.hasNext();) { + Operator opR = itR.next(); + if (opL.logicalEqualsTree(opR)) { + itL.remove(); + itR.remove(); + break; + } + } + } + return opsL.isEmpty() && opsR.isEmpty(); + } + + private List getRootOps(PlanMapper pmL) { + List ops = pmL.getAll(Operator.class); + for (Iterator iterator = ops.iterator(); iterator.hasNext();) { + Operator o = iterator.next(); + if (o.getNumChild() != 0) { + iterator.remove(); + } + } + return ops; + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/stats/OperatorStats.java ql/src/java/org/apache/hadoop/hive/ql/stats/OperatorStats.java new file mode 100644 index 0000000000..52e18a8030 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/stats/OperatorStats.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.stats; + +public class OperatorStats { + private final String operatorId; + private long outputRecords; + + public OperatorStats(final String opId) { + this.operatorId = opId; + this.outputRecords = -1; + } + + public long getOutputRecords() { + return outputRecords; + } + + public void setOutputRecords(final long outputRecords) { + this.outputRecords = outputRecords; + } + + public String getOperatorId() { + return operatorId; + } + + @Override + public String toString() { + return String.format("OperatorStats %s records: %d", operatorId, outputRecords); + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/stats/OperatorStatsReaderHook.java ql/src/java/org/apache/hadoop/hive/ql/stats/OperatorStatsReaderHook.java new file mode 100644 index 0000000000..9a3241975c --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/stats/OperatorStatsReaderHook.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.stats; + +import java.util.List; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.QueryPlan; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.exec.tez.TezTask; +import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext; +import org.apache.hadoop.hive.ql.hooks.HookContext; +import org.apache.hadoop.hive.ql.hooks.HookContext.HookType; +import org.apache.hadoop.hive.ql.hooks.PrivateHookContext; +import org.apache.hadoop.hive.ql.plan.BaseWork; +import org.apache.hadoop.hive.ql.plan.OperatorDesc; +import org.apache.tez.common.counters.TezCounter; +import org.apache.tez.common.counters.TezCounters; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class OperatorStatsReaderHook implements ExecuteWithHookContext { + + private static final Logger LOG = LoggerFactory.getLogger(OperatorStatsReaderHook.class); + + @Override + public void run(HookContext hookContext) throws Exception { + + if (hookContext.getHookType() == HookType.PRE_EXEC_HOOK) { + return; + } + + HiveConf conf = hookContext.getConf(); + QueryPlan plan = hookContext.getQueryPlan(); + List rootTasks = Utilities.getTezTasks(plan.getRootTasks()); + for (TezTask tezTask : rootTasks) { + List baseWorks = tezTask.getWork().getAllWork(); + for (BaseWork baseWork : baseWorks) { + String vertexName = baseWork.getName(); + LOG.debug("Reading runtime statistics for tez vertex task: {}", vertexName); + TezCounters counters = tezTask.getTezCounters(); + if (counters != null) { + String groupName = HiveConf.getVar(conf, HiveConf.ConfVars.HIVECOUNTERGROUP); + for (Operator op : baseWork.getAllOperators()) { + String operatorId = op.getOperatorId(); + OperatorStats operatorStats = null; + String counterName = Operator.Counter.RECORDS_OUT_OPERATOR.toString() + "_" + operatorId; + TezCounter tezCounter = counters.getGroup(groupName).findCounter(counterName, false); + if (tezCounter != null) { + if (operatorStats == null) { + operatorStats = new OperatorStats(operatorId); + } + operatorStats.setOutputRecords(tezCounter.getValue()); + } + + if (operatorStats != null) { + ((PrivateHookContext) hookContext).getContext().getPlanMapper().link(op, operatorStats); + } else { + LOG.debug("Unable to get statistics for vertex: {} opId: {} groupName: {}", vertexName, operatorId, + groupName); + } + } + } + } + } + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAssertTrueOOM.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAssertTrueOOM.java new file mode 100644 index 0000000000..c5c73835af --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAssertTrueOOM.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.exec.mapjoin.MapJoinMemoryExhaustionError; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.ql.udf.UDFType; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.BooleanWritable; + +@UDFType(deterministic = false) +public class GenericUDFAssertTrueOOM extends GenericUDF { + private ObjectInspectorConverters.Converter conditionConverter = null; + + public GenericUDFAssertTrueOOM() { + } + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + HiveConf conf = SessionState.getSessionConf(); + if (!conf.getBoolVar(HiveConf.ConfVars.HIVE_IN_TEST)) { + throw new RuntimeException("this UDF is only available in testmode"); + } + if (arguments.length != 1) { + throw new UDFArgumentLengthException("ASSERT_TRUE_OOM() expects one argument."); + } + if (arguments[0].getCategory() != Category.PRIMITIVE) { + throw new UDFArgumentTypeException(0, "Argument to ASSERT_TRUE_OOM() should be primitive."); + } + conditionConverter = + ObjectInspectorConverters.getConverter(arguments[0], PrimitiveObjectInspectorFactory.writableBooleanObjectInspector); + + return PrimitiveObjectInspectorFactory.writableVoidObjectInspector; + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + BooleanWritable condition = (BooleanWritable) conditionConverter.convert(arguments[0].get()); + if (condition == null || !condition.get()) { + throw new MapJoinMemoryExhaustionError("assert_true_oom: assertation failed; Simulated OOM"); + } + return null; + } + + @Override + public String getDisplayString(String[] children) { + return getStandardDisplayString("assert_true_oom", children); + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEnforceNotNullConstraint.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEnforceNotNullConstraint.java index 6c469bf944..e8516fb8e0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEnforceNotNullConstraint.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEnforceNotNullConstraint.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java index fa3f89fda1..e65288b01d 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java @@ -41,8 +41,6 @@ import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.OperatorFactory; -import org.apache.hadoop.hive.ql.exec.util.collectoroperator.RowVectorCollectorTestOperator; -import org.apache.hadoop.hive.ql.exec.util.rowobjects.RowTestObjects; import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorUDAFCountStar; import org.apache.hadoop.hive.ql.exec.vector.util.FakeCaptureVectorToRowOutputOperator; import org.apache.hadoop.hive.ql.exec.vector.util.FakeVectorRowBatchFromConcat; @@ -61,7 +59,6 @@ import org.apache.hadoop.hive.ql.plan.VectorGroupByDesc.ProcessingMode; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFCount; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFCount.GenericUDAFCountEvaluator; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFMax; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFMin; @@ -75,7 +72,6 @@ import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -2555,7 +2551,7 @@ public void testAggregateLongIterable ( (VectorGroupByOperator) Vectorizer.vectorizeGroupByOperator(groupByOp, ctx, vectorDesc); FakeCaptureVectorToRowOutputOperator out = FakeCaptureVectorToRowOutputOperator.addCaptureOutputChild(cCtx, vgo); - vgo.initialize(null, null); + vgo.initialize(hconf, null); for (VectorizedRowBatch unit: data) { vgo.process(unit, 0); diff --git ql/src/test/org/apache/hadoop/hive/ql/optimizer/signature/TestOpSigFactory.java ql/src/test/org/apache/hadoop/hive/ql/optimizer/signature/TestOpSigFactory.java new file mode 100644 index 0000000000..9c4fc43ef7 --- /dev/null +++ ql/src/test/org/apache/hadoop/hive/ql/optimizer/signature/TestOpSigFactory.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.optimizer.signature; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import org.apache.hadoop.hive.ql.CompilationOpContext; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.plan.AbstractOperatorDesc; +import org.apache.hadoop.hive.ql.plan.Signature; +import org.apache.hadoop.hive.ql.plan.api.OperatorType; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + + +public class TestOpSigFactory { + CompilationOpContext cCtx = new CompilationOpContext(); + + @Rule + public MockitoRule a = MockitoJUnit.rule(); + + @Spy + OpTreeSignatureFactory f = OpTreeSignatureFactory.newCache(); + + public static class SampleDesc extends AbstractOperatorDesc { + private static final long serialVersionUID = 1L; + private int desc_invocations; + + @Signature + public int asd() { + desc_invocations++; + return 8; + } + + public int getDesc_invocations() { + return desc_invocations; + } + } + + static class SampleOperator extends Operator { + private static final long serialVersionUID = 1L; + + @Override + public void process(Object row, int tag) throws HiveException { + } + + @Override + public String getName() { + return "A1"; + } + + @Override + public OperatorType getType() { + return OperatorType.FILTER; + } + } + + @Test + public void checkExplicit() { + SampleOperator so = new SampleOperator(); + SampleDesc sd = new SampleDesc(); + so.setConf(sd); + + f.getSignature(so); + f.getSignature(so); + + verify(f, times(2)).getSignature(Mockito.any()); + assertEquals(1, sd.getDesc_invocations()); + } + + @Test + public void checkImplicit() { + SampleOperator so = new SampleOperator(); + SampleDesc sd = new SampleDesc(); + so.setConf(sd); + + SampleOperator so2 = new SampleOperator(); + SampleDesc sd2 = new SampleDesc(); + so2.setConf(sd2); + + so.getParentOperators().add(so2); + so2.getChildOperators().add(so); + + f.getSignature(so); + // computes the sig of every object + verify(f, times(2)).getSignature(Mockito.any()); + assertEquals(1, sd.getDesc_invocations()); + assertEquals(1, sd2.getDesc_invocations()); + + f.getSignature(so); + f.getSignature(so2); + + verify(f, times(4)).getSignature(Mockito.any()); + assertEquals(1, sd.getDesc_invocations()); + assertEquals(1, sd2.getDesc_invocations()); + } + +} diff --git ql/src/test/org/apache/hadoop/hive/ql/optimizer/signature/TestOperatorSignature.java ql/src/test/org/apache/hadoop/hive/ql/optimizer/signature/TestOperatorSignature.java new file mode 100644 index 0000000000..ee7a3fba9e --- /dev/null +++ ql/src/test/org/apache/hadoop/hive/ql/optimizer/signature/TestOperatorSignature.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.optimizer.signature; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import java.util.List; + +import org.apache.hadoop.hive.ql.CompilationOpContext; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.OperatorFactory; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; +import org.apache.hadoop.hive.ql.plan.FilterDesc; +import org.apache.hadoop.hive.ql.plan.OperatorDesc; +import org.apache.hadoop.hive.ql.plan.TableScanDesc; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFConcat; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.junit.Test; + +import com.google.common.collect.Lists; + +public class TestOperatorSignature { + + GenericUDF udf = new GenericUDFConcat(); + + CompilationOpContext cCtx = new CompilationOpContext(); + + @Test + public void testFilterOpEquals() { + Operator op7 = getFilterOp(7); + Operator op8 = getFilterOp(8); + Operator op7b = getFilterOp(7); + + checkEquals(op7, op7b); + checkNotEquals(op7, op8); + } + + @Test + public void testTree1() { + Operator tr37 = getFilTsOp(3, 7); + Operator tr37a = getFilTsOp(3, 7); + Operator tr17 = getFilTsOp(1, 7); + Operator tr31 = getFilTsOp(3, 1); + + checkEquals(tr37, tr37a); + + checkTreeNotEquals(tr37, tr17); + checkTreeEquals(tr37, tr37a); + checkTreeNotEquals(tr37, tr31); + } + + private Operator getFilTsOp(int i, int j) { + Operator ts = getTsOp(i); + Operator fil = getFilterOp(j); + + ts.getChildOperators().add(fil); + fil.getParentOperators().add(ts); + + return fil; + } + + @Test + public void testTableScand() { + Operator t1 = getTsOp(3); + Operator t1a = getTsOp(3); + Operator t2 = getTsOp(4); + + assertTrue(t1.logicalEquals(t1a)); + checkEquals(t1, t1a); + checkNotEquals(t1, t2); + } + + private void checkEquals(Operator o1, Operator o2) { + assertTrue(o1.logicalEquals(o2)); + OpSignature s1 = OpSignature.of(o1); + OpSignature s2 = OpSignature.of(o2); + + s1.proveEquals(s2); + assertTrue("sigCmp", s1.signatureCompare(s2)); + assertEquals(s1.hashCode(), s2.hashCode()); + assertEquals(s1, s2); + } + + + private void checkNotEquals(Operator o1, Operator o2) { + assertFalse(o1.logicalEquals(o2)); + OpSignature s1 = OpSignature.of(o1); + OpSignature s2 = OpSignature.of(o2); + + assertFalse(s1.signatureCompare(s2)); + // this might be a little bit too much...but in most cases this should be true + assertNotEquals(s1.hashCode(), s2.hashCode()); + assertNotEquals(s1, s2); + } + + private void checkTreeEquals(Operator o1, Operator o2) { + OpTreeSignature ts1 = OpTreeSignature.of(o1); + OpTreeSignature ts2 = OpTreeSignature.of(o2); + + assertEquals(ts1.hashCode(), ts2.hashCode()); + assertEquals(ts1, ts2); + } + + private void checkTreeNotEquals(Operator o1, Operator o2) { + + OpTreeSignature ts1 = OpTreeSignature.of(o1); + OpTreeSignature ts2 = OpTreeSignature.of(o2); + + // FIXME: currently ExprNode-s are not in the signature; which cause this to be the same + assertNotEquals(ts1.hashCode(), ts2.hashCode()); + assertNotEquals(ts1, ts2); + } + + + private Operator getFilterOp(int constVal) { + ExprNodeDesc pred = new ExprNodeConstantDesc(constVal); + FilterDesc fd = new FilterDesc(pred, true); + Operator op = OperatorFactory.get(cCtx, fd); + return op; + } + + private Operator getTsOp(int i) { + Table tblMetadata = new Table("db", "table"); + // FIXME: I think this shouldn't be sensitive to the alias; but currently its included in logicalEquals...check that + TableScanDesc desc = new TableScanDesc("alias"/*+ cCtx.nextOperatorId()*/, tblMetadata); + List as = + Lists.newArrayList(new ExprNodeConstantDesc(TypeInfoFactory.intTypeInfo, Integer.valueOf(i)), + new ExprNodeColumnDesc(TypeInfoFactory.intTypeInfo, "c1", "aa", false)); + ExprNodeGenericFuncDesc f1 = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo, udf, as); + desc.setFilterExpr(f1); + Operator ts = OperatorFactory.get(cCtx, desc); + return ts; + } + + +} diff --git ql/src/test/org/apache/hadoop/hive/ql/plan/mapping/TestCounterMapping.java ql/src/test/org/apache/hadoop/hive/ql/plan/mapping/TestCounterMapping.java new file mode 100644 index 0000000000..fcb6284e01 --- /dev/null +++ ql/src/test/org/apache/hadoop/hive/ql/plan/mapping/TestCounterMapping.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.plan.mapping; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.DriverFactory; +import org.apache.hadoop.hive.ql.IDriver; +import org.apache.hadoop.hive.ql.exec.FilterOperator; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; +import org.apache.hadoop.hive.ql.parse.ParseException; +import org.apache.hadoop.hive.ql.plan.mapper.PlanMapper; +import org.apache.hadoop.hive.ql.plan.mapper.SimpleRuntimeStatsSource; +import org.apache.hadoop.hive.ql.plan.mapper.PlanMapper.LinkGroup; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.ql.stats.OperatorStats; +import org.apache.hadoop.hive.ql.stats.OperatorStatsReaderHook; +import org.apache.hive.testutils.HiveTestEnvSetup; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +public class TestCounterMapping { + + @ClassRule + public static HiveTestEnvSetup env_setup = new HiveTestEnvSetup(); + + @Rule + public TestRule methodRule = env_setup.getMethodRule(); + + static Comparator> OPERATOR_ID_COMPARATOR = new Comparator>() { + + @Override + public int compare(Operator o1, Operator o2) { + return Objects.compare(o1.getOperatorId(), o2.getOperatorId(), Comparator.naturalOrder()); + } + }; + + + @BeforeClass + public static void beforeClass() throws Exception { + IDriver driver = createDriver(); + dropTables(driver); + String cmds[] = { + // @formatter:off + "create table s (x int)", + "insert into s values (1),(2),(3),(4),(5),(6),(7),(8),(9),(10)", + "create table tu(id_uv int,id_uw int,u int)", + "create table tv(id_uv int,v int)", + "create table tw(id_uw int,w int)", + + "from s\n" + + "insert overwrite table tu\n" + + " select x,x,x\n" + + " where x<=6 or x=10\n" + + "insert overwrite table tv\n" + + " select x,x\n" + + " where x<=3 or x=10\n" + + "insert overwrite table tw\n" + + " select x,x\n" + + "", + // @formatter:on + }; + for (String cmd : cmds) { + int ret = driver.run(cmd).getResponseCode(); + assertEquals("Checking command success", 0, ret); + } + } + + @AfterClass + public static void afterClass() throws Exception { + IDriver driver = createDriver(); + dropTables(driver); + } + + public static void dropTables(IDriver driver) throws Exception { + String tables[] = { "s", "tu", "tv", "tw" }; + for (String t : tables) { + int ret = driver.run("drop table if exists " + t).getResponseCode(); + assertEquals("Checking command success", 0, ret); + } + } + + private PlanMapper getMapperForQuery(IDriver driver, String query) { + int ret; + ret = driver.run(query).getResponseCode(); + assertEquals("Checking command success", 0, ret); + PlanMapper pm0 = driver.getContext().getPlanMapper(); + return pm0; + } + + @Test + public void testUsageOfRuntimeInfo() throws ParseException { + IDriver driver = createDriver(); + String query = "select sum(u) from tu where u>1"; + PlanMapper pm1 = getMapperForQuery(driver, query); + + List filters1 = pm1.getAll(FilterOperator.class); + filters1.sort(OPERATOR_ID_COMPARATOR.reversed()); + FilterOperator filter1 = filters1.get(0); + + ((Driver) driver).setRuntimeStatsSource(new SimpleRuntimeStatsSource(pm1)); + + PlanMapper pm2 = getMapperForQuery(driver, query); + + List filters2 = pm2.getAll(FilterOperator.class); + filters2.sort(OPERATOR_ID_COMPARATOR.reversed()); + FilterOperator filter2 = filters2.get(0); + + assertEquals("original check", 7, filter1.getStatistics().getNumRows()); + assertEquals("optimized check", 6, filter2.getStatistics().getNumRows()); + + } + + @Test + @Ignore("needs HiveFilter mapping") + public void testMappingJoinLookup() throws ParseException { + IDriver driver = createDriver(); + + PlanMapper pm0 = getMapperForQuery(driver, "select sum(tu.id_uv),sum(u) from tu join tv on (tu.id_uv = tv.id_uv) where u>1 and v>1"); + + Iterator itG = pm0.iterateGroups(); + int checkedOperators = 0; + // FIXME: introduce the Operator trimmer mapper! + while (itG.hasNext()) { + LinkGroup g = itG.next(); + List hfs = g.getAll(HiveFilter.class); + List oss = g.getAll(OperatorStats.class); + List fos = g.getAll(FilterOperator.class); + + if (fos.size() > 0 && oss.size() > 0) { + if (hfs.size() == 0) { + fail("HiveFilter is not connected?"); + } + OperatorStats os = oss.get(0); + if (!(os.getOutputRecords() == 3 || os.getOutputRecords() == 6)) { + fail("nonexpected number of records produced"); + } + checkedOperators++; + } + } + assertEquals(2, checkedOperators); + } + + private static IDriver createDriver() { + // HiveConf conf = new HiveConf(Driver.class); + HiveConf conf = env_setup.getTestCtx().hiveConf; + conf.setVar(ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES, "disabled"); + conf.set("hive.auto.convert.join", "false"); + conf.set("hive.optimize.ppd", "false"); + + conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, + "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); + // conf.setVar(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK, CheckInputReadEntityDirect.class.getName()); + HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); + HiveConf.setVar(conf, HiveConf.ConfVars.POSTEXECHOOKS, OperatorStatsReaderHook.class.getName()); + SessionState.start(conf); + + IDriver driver = DriverFactory.newDriver(conf); + return driver; + } + + +} diff --git ql/src/test/org/apache/hadoop/hive/ql/plan/mapping/TestOperatorCmp.java ql/src/test/org/apache/hadoop/hive/ql/plan/mapping/TestOperatorCmp.java new file mode 100644 index 0000000000..71e32b24ae --- /dev/null +++ ql/src/test/org/apache/hadoop/hive/ql/plan/mapping/TestOperatorCmp.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.plan.mapping; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.Iterator; +import java.util.List; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.ql.DriverFactory; +import org.apache.hadoop.hive.ql.IDriver; +import org.apache.hadoop.hive.ql.exec.FilterOperator; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.TableScanOperator; +import org.apache.hadoop.hive.ql.parse.ParseException; +import org.apache.hadoop.hive.ql.plan.mapper.PlanMapper; +import org.apache.hadoop.hive.ql.plan.mapper.PlanMapper.LinkGroup; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.ql.stats.OperatorStatsReaderHook; +import org.apache.hive.testutils.HiveTestEnvSetup; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +public class TestOperatorCmp { + + @ClassRule + public static HiveTestEnvSetup env_setup = new HiveTestEnvSetup(); + + @Rule + public TestRule methodRule = env_setup.getMethodRule(); + + @BeforeClass + public static void beforeClass() throws Exception { + IDriver driver = createDriver(); + dropTables(driver); + String cmds[] = { + // @formatter:off + "create table tu(id_uv int,id_uw int,u int)", + "create table tv(id_uv int,v int)", + "create table tw(id_uw int,w int)", + + "insert into tu values (10,10,10),(1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5),(6,6,6)", + "insert into tv values (10,10),(1,1),(2,2),(3,3)", + "insert into tw values (10,10),(1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8),(9,9)", + // @formatter:on + }; + for (String cmd : cmds) { + int ret = driver.run(cmd).getResponseCode(); + assertEquals("Checking command success", 0, ret); + } + } + + @AfterClass + public static void afterClass() throws Exception { + IDriver driver = createDriver(); + dropTables(driver); + } + + public static void dropTables(IDriver driver) throws Exception { + String tables[] = { "tu", "tv", "tw" }; + for (String t : tables) { + int ret = driver.run("drop table if exists " + t).getResponseCode(); + assertEquals("Checking command success", 0, ret); + } + } + + private PlanMapper getMapperForQuery(IDriver driver, String query) { + int ret = driver.run(query).getResponseCode(); + assertEquals("Checking command success", 0, ret); + PlanMapper pm0 = driver.getContext().getPlanMapper(); + return pm0; + } + + @Test + public void testUnrelatedFiltersAreNotMatched0() throws ParseException { + IDriver driver = createDriver(); + String query = "select u from tu where id_uv = 1 union all select v from tv where id_uv = 1"; + + PlanMapper pm = getMapperForQuery(driver, query); + Iterator itG = pm.iterateGroups(); + List fos = pm.getAll(FilterOperator.class); + assertEquals(2, fos.size()); + + assertFalse("logicalEquals", compareOperators(fos.get(0), fos.get(1))); + + } + + @Test + public void testUnrelatedFiltersAreNotMatched1() throws ParseException { + IDriver driver = createDriver(); + PlanMapper pm0 = getMapperForQuery(driver, "select u from tu where id_uv = 1 group by u"); + PlanMapper pm1 = getMapperForQuery(driver, "select v from tv where id_uv = 1 group by v"); + List fos0 = pm0.getAll(FilterOperator.class); + List fos1 = pm1.getAll(FilterOperator.class); + assertEquals(1, fos0.size()); + assertEquals(1, fos1.size()); + + assertFalse("logicalEquals", compareOperators(fos0.get(0), fos1.get(0))); + } + + @Test + public void testDifferentFiltersAreNotMatched1() throws ParseException { + IDriver driver = createDriver(); + PlanMapper pm0 = getMapperForQuery(driver, "select u from tu where id_uv = 1 group by u"); + PlanMapper pm1 = getMapperForQuery(driver, "select u from tu where id_uv = 2 group by u"); + + assertHelper(AssertHelperOp.SAME, pm0, pm1, TableScanOperator.class); + assertHelper(AssertHelperOp.NOT_SAME, pm0, pm1, FilterOperator.class); + + } + + @Test + public void testSameFiltersMatched1() throws ParseException, Exception { + IDriver driver = createDriver(); + PlanMapper pm0 = getMapperForQuery(driver, "select u from tu where id_uv = 1 group by u"); + PlanMapper pm1 = getMapperForQuery(driver, "select u from tu where id_uv = 1 group by u"); + + assertHelper(AssertHelperOp.SAME, pm0, pm1, FilterOperator.class); + assertHelper(AssertHelperOp.SAME, pm0, pm1, TableScanOperator.class); + } + + + enum AssertHelperOp { + SAME, NOT_SAME + }; + + private > void assertHelper(AssertHelperOp same,PlanMapper pm0, PlanMapper pm1, Class clazz) { + List fos0 = pm0.getAll(clazz); + List fos1 = pm1.getAll(clazz); + assertEquals(1, fos0.size()); + assertEquals(1, fos1.size()); + + if (same == AssertHelperOp.SAME) { + assertTrue(clazz + " " + same, compareOperators(fos0.get(0), fos1.get(0))); + } else { + assertFalse(clazz + " " + same, compareOperators(fos0.get(0), fos1.get(0))); + } + } + + private boolean compareOperators(Operator opL, Operator opR) { + return opL.logicalEqualsTree(opR); + } + + private static IDriver createDriver() { + HiveConf conf = env_setup.getTestCtx().hiveConf; + + conf.setBoolVar(ConfVars.HIVE_QUERY_REEXECUTION_ENABLED, true); + conf.setVar(ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES, "reoptimize"); + conf.set("zzz", "1"); + conf.set("reexec.overlay.zzz", "2000"); + // + conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, + "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); + HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); + HiveConf.setVar(conf, HiveConf.ConfVars.POSTEXECHOOKS, OperatorStatsReaderHook.class.getName()); + SessionState.start(conf); + + IDriver driver = DriverFactory.newDriver(conf); + return driver; + } + + +} diff --git ql/src/test/org/apache/hadoop/hive/ql/plan/mapping/TestReOptimization.java ql/src/test/org/apache/hadoop/hive/ql/plan/mapping/TestReOptimization.java new file mode 100644 index 0000000000..6d7bb07dc1 --- /dev/null +++ ql/src/test/org/apache/hadoop/hive/ql/plan/mapping/TestReOptimization.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.plan.mapping; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.ql.DriverFactory; +import org.apache.hadoop.hive.ql.IDriver; +import org.apache.hadoop.hive.ql.exec.FilterOperator; +import org.apache.hadoop.hive.ql.plan.Statistics; +import org.apache.hadoop.hive.ql.plan.mapper.PlanMapper; +import org.apache.hadoop.hive.ql.plan.mapper.PlanMapper.LinkGroup; +import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.ql.stats.OperatorStats; +import org.apache.hadoop.hive.ql.stats.OperatorStatsReaderHook; +import org.apache.hive.testutils.HiveTestEnvSetup; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +public class TestReOptimization { + + @ClassRule + public static HiveTestEnvSetup env_setup = new HiveTestEnvSetup(); + + @Rule + public TestRule methodRule = env_setup.getMethodRule(); + + @BeforeClass + public static void beforeClass() throws Exception { + IDriver driver = createDriver(""); + dropTables(driver); + String cmds[] = { + // @formatter:off + "create table tu(id_uv int,id_uw int,u int)", + "create table tv(id_uv int,v int)", + "create table tw(id_uw int,w int)", + + "insert into tu values (10,10,10),(1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5),(6,6,6)", + "insert into tv values (10,10),(1,1),(2,2),(3,3)", + "insert into tw values (10,10),(1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8),(9,9)", + // @formatter:on + }; + for (String cmd : cmds) { + int ret = driver.run(cmd).getResponseCode(); + assertEquals("Checking command success", 0, ret); + } + } + + @AfterClass + public static void afterClass() throws Exception { + IDriver driver = createDriver(""); + dropTables(driver); + } + + public static void dropTables(IDriver driver) throws Exception { + String tables[] = { "tu", "tv", "tw" }; + for (String t : tables) { + int ret = driver.run("drop table if exists " + t).getResponseCode(); + assertEquals("Checking command success", 0, ret); + } + } + + private PlanMapper getMapperForQuery(IDriver driver, String query) throws CommandProcessorResponse { + CommandProcessorResponse res = driver.run(query); + if (res.getResponseCode() != 0) { + throw res; + } + PlanMapper pm0 = driver.getContext().getPlanMapper(); + return pm0; + } + + @Test + public void testStatsAreSetInReopt() throws Exception { + IDriver driver = createDriver("overlay,reoptimize"); + String query = "select assert_true_oom(${hiveconf:zzz} > sum(u*v)) from tu join tv on (tu.id_uv=tv.id_uv) where u<10 and v>1"; + + PlanMapper pm = getMapperForQuery(driver, query); + Iterator itG = pm.iterateGroups(); + int checkedOperators = 0; + while (itG.hasNext()) { + LinkGroup g = itG.next(); + List fos = g.getAll(FilterOperator.class); + List oss = g.getAll(OperatorStats.class); + // FIXME: oss seems to contain duplicates + + if (fos.size() > 0 && oss.size() > 0) { + fos.sort(TestCounterMapping.OPERATOR_ID_COMPARATOR.reversed()); + + FilterOperator fo = fos.get(0); + OperatorStats os = oss.get(0); + + Statistics stats = fo.getStatistics(); + assertEquals(os.getOutputRecords(), stats.getNumRows()); + + if (!(os.getOutputRecords() == 3 || os.getOutputRecords() == 6)) { + fail("nonexpected number of records produced"); + } + checkedOperators++; + } + } + assertEquals(2, checkedOperators); + } + + @Test + public void testReExecutedIfMapJoinError() throws Exception { + + IDriver driver = createDriver("overlay,reoptimize"); + String query = + "select assert_true_oom(${hiveconf:zzz}>sum(1)) from tu join tv on (tu.id_uv=tv.id_uv) where u<10 and v>1"; + PlanMapper pm = getMapperForQuery(driver, query); + + } + + @Test(expected = CommandProcessorResponse.class) + public void testNotReExecutedIfAssertionError() throws Exception { + IDriver driver = createDriver("reoptimize"); + String query = + "select assert_true(${hiveconf:zzz}>sum(1)) from tu join tv on (tu.id_uv=tv.id_uv) where u<10 and v>1"; + + PlanMapper pm = getMapperForQuery(driver, query); + assertEquals(1, driver.getContext().getExecutionIndex()); + } + + @Test + public void testExplainSupport() throws Exception { + + IDriver driver = createDriver("overlay,reoptimize"); + String query = "explain reoptimization select 1 from tu join tv on (tu.id_uv=tv.id_uv) where u<10 and v>1"; + PlanMapper pm = getMapperForQuery(driver, query); + List res = new ArrayList<>(); + List res1 = new ArrayList<>(); + while (driver.getResults(res1)) { + res.addAll(res1); + } + + assertEquals("2TS", 2, res.stream().filter(line -> line.contains("TS_")).count()); + assertEquals("2TS(runtime)", 2, + res.stream().filter(line -> line.contains("TS") && line.contains("runtime")).count()); + + } + + private static IDriver createDriver(String strategies) { + HiveConf conf = env_setup.getTestCtx().hiveConf; + + conf.setBoolVar(ConfVars.HIVE_QUERY_REEXECUTION_ENABLED, true); + conf.setVar(ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES, strategies); + conf.setBoolVar(ConfVars.HIVE_EXPLAIN_USER, true); + conf.set("zzz", "1"); + conf.set("reexec.overlay.zzz", "2000"); + // + conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, + "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); + HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); + HiveConf.setVar(conf, HiveConf.ConfVars.POSTEXECHOOKS, OperatorStatsReaderHook.class.getName()); + SessionState.start(conf); + + IDriver driver = DriverFactory.newDriver(conf); + return driver; + } + + +} diff --git ql/src/test/org/apache/hive/testutils/HiveTestEnvSetup.java ql/src/test/org/apache/hive/testutils/HiveTestEnvSetup.java index cc97317f55..f872da02a3 100644 --- ql/src/test/org/apache/hive/testutils/HiveTestEnvSetup.java +++ ql/src/test/org/apache/hive/testutils/HiveTestEnvSetup.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git ql/src/test/org/apache/hive/testutils/TestHiveTestEnvSetup.java ql/src/test/org/apache/hive/testutils/TestHiveTestEnvSetup.java index 91613667f8..c34affd2c4 100644 --- ql/src/test/org/apache/hive/testutils/TestHiveTestEnvSetup.java +++ ql/src/test/org/apache/hive/testutils/TestHiveTestEnvSetup.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git ql/src/test/org/apache/hive/testutils/TestX1.java ql/src/test/org/apache/hive/testutils/TestX1.java new file mode 100644 index 0000000000..fea4151923 --- /dev/null +++ ql/src/test/org/apache/hive/testutils/TestX1.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.testutils; + +import static org.junit.Assert.assertEquals; +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.DriverFactory; +import org.apache.hadoop.hive.ql.IDriver; +import org.apache.hadoop.hive.ql.parse.ParseException; +import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hive.testutils.HiveTestEnvSetup; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +public class TestX1 { + + @ClassRule + public static HiveTestEnvSetup env_setup = new HiveTestEnvSetup(); + + @Rule + public TestRule methodRule = env_setup.getMethodRule(); + + @BeforeClass + public static void beforeClass() throws Exception { + IDriver driver = createDriver(); + dropTables(driver); + String cmds[] = { + // @formatter:off + "create table tu(u int)", + "insert into tu values (1),(2),(3),(100)", + "create table tv(u int)", + "insert into tv values (4),(100)", + // @formatter:on + }; + for (String cmd : cmds) { + int ret = driver.run(cmd).getResponseCode(); + assertEquals("Checking command success", 0, ret); + } + } + + @AfterClass + public static void afterClass() throws Exception { + IDriver driver = createDriver(); + dropTables(driver); + } + + public static void dropTables(IDriver driver) throws Exception { + String tables[] = { "s", "tu", "tv", "tw" }; + for (String t : tables) { + int ret = driver.run("drop table if exists " + t).getResponseCode(); + assertEquals("Checking command success", 0, ret); + } + } + + @Test + public void testMappingSameQuery() throws ParseException, Exception { + IDriver driver = createDriver(); + String query = "select u from tu where u < 100 union all select u from tv where u < 100 "; + CommandProcessorResponse ret = driver.run(query); + assertEquals(0, ret.getResponseCode()); + + List res = new ArrayList(); + driver.getFetchTask().fetch(res); + System.out.println(res); + assertEquals(4, res.size()); + } + + private static IDriver createDriver() { + HiveConf conf = env_setup.getTestCtx().hiveConf; + conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, + "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); + HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); + SessionState.start(conf); + + IDriver driver = DriverFactory.newDriver(conf); + return driver; + } + +} diff --git ql/src/test/queries/clientpositive/retry_failure.q ql/src/test/queries/clientpositive/retry_failure.q new file mode 100644 index 0000000000..0b9922b66d --- /dev/null +++ ql/src/test/queries/clientpositive/retry_failure.q @@ -0,0 +1,11 @@ +create table tx(a int,f string); +insert into tx values (1,'non_existent_file'); + +set zzz=1; +set reexec.overlay.zzz=2; + +set hive.query.reexecution.enabled=true; +set hive.query.reexecution.strategies=overlay; + +select assert_true(${hiveconf:zzz} > a) from tx group by a; + diff --git ql/src/test/queries/clientpositive/retry_failure_oom.q ql/src/test/queries/clientpositive/retry_failure_oom.q new file mode 100644 index 0000000000..21c9f8c641 --- /dev/null +++ ql/src/test/queries/clientpositive/retry_failure_oom.q @@ -0,0 +1,12 @@ + +create table tx(a int,f string); +insert into tx values (1,'non_existent_file'); + +set zzz=1; +set reexec.overlay.zzz=2; + +set hive.query.reexecution.enabled=true; +set hive.query.reexecution.strategies=overlay,reoptimize; + +select assert_true_oom(${hiveconf:zzz} > a) from tx group by a; + diff --git ql/src/test/queries/clientpositive/retry_failure_stat_changes.q ql/src/test/queries/clientpositive/retry_failure_stat_changes.q new file mode 100644 index 0000000000..c5942558ec --- /dev/null +++ ql/src/test/queries/clientpositive/retry_failure_stat_changes.q @@ -0,0 +1,29 @@ + +create table tx(a int,u int); +insert into tx values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(10,10); + +create table px(a int,p int); +insert into px values (2,2),(3,3),(5,5),(7,7),(11,11); + + +set hive.explain.user=true; +set hive.query.reexecution.enabled=true; +set hive.query.reexecution.strategies=overlay,reoptimize; + +explain REOPTIMIZATION +select sum(u*p) from tx join px on (u=p) where u<10 and p>2; + +set hive.auto.convert.join=false; +explain analyze +select sum(u*p) from tx join px on (u=p) where u<10 and p>2; +set hive.auto.convert.join=true; +explain analyze +select sum(u*p) from tx join px on (u=p) where u<10 and p>2; + +set zzz=1; +set reexec.overlay.zzz=2000; + +explain +select assert_true_oom(${hiveconf:zzz} > sum(u*p)) from tx join px on (tx.a=px.a) where u<10 and p>2; +select assert_true_oom(${hiveconf:zzz} > sum(u*p)) from tx join px on (tx.a=px.a) where u<10 and p>2; + diff --git ql/src/test/results/clientpositive/llap/dp_counter_mm.q.out ql/src/test/results/clientpositive/llap/dp_counter_mm.q.out index 869916078b..18f4c69a19 100644 --- ql/src/test/results/clientpositive/llap/dp_counter_mm.q.out +++ ql/src/test/results/clientpositive/llap/dp_counter_mm.q.out @@ -23,6 +23,18 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 57 RECORDS_OUT_1_default.src2: 84 RECORDS_OUT_INTERMEDIATE_Map_1: 57 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_5: 84 + RECORDS_OUT_OPERATOR_FS_4: 84 + RECORDS_OUT_OPERATOR_FS_6: 57 + RECORDS_OUT_OPERATOR_GBY_2: 57 + RECORDS_OUT_OPERATOR_GBY_4: 57 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 57 + RECORDS_OUT_OPERATOR_SEL_1: 84 + RECORDS_OUT_OPERATOR_SEL_2: 84 + RECORDS_OUT_OPERATOR_SEL_5: 57 + RECORDS_OUT_OPERATOR_TS_0: 500 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -41,6 +53,18 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 121 RECORDS_OUT_1_default.src2: 189 RECORDS_OUT_INTERMEDIATE_Map_1: 121 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_5: 189 + RECORDS_OUT_OPERATOR_FS_4: 189 + RECORDS_OUT_OPERATOR_FS_6: 121 + RECORDS_OUT_OPERATOR_GBY_2: 121 + RECORDS_OUT_OPERATOR_GBY_4: 121 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 121 + RECORDS_OUT_OPERATOR_SEL_1: 189 + RECORDS_OUT_OPERATOR_SEL_2: 189 + RECORDS_OUT_OPERATOR_SEL_5: 121 + RECORDS_OUT_OPERATOR_TS_0: 500 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -67,6 +91,18 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 121 RECORDS_OUT_1_default.src2: 189 RECORDS_OUT_INTERMEDIATE_Map_1: 121 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_5: 189 + RECORDS_OUT_OPERATOR_FS_4: 189 + RECORDS_OUT_OPERATOR_FS_6: 121 + RECORDS_OUT_OPERATOR_GBY_2: 121 + RECORDS_OUT_OPERATOR_GBY_4: 121 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 121 + RECORDS_OUT_OPERATOR_SEL_1: 189 + RECORDS_OUT_OPERATOR_SEL_2: 189 + RECORDS_OUT_OPERATOR_SEL_5: 121 + RECORDS_OUT_OPERATOR_TS_0: 500 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -85,6 +121,18 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 184 RECORDS_OUT_1_default.src2: 292 RECORDS_OUT_INTERMEDIATE_Map_1: 184 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_5: 292 + RECORDS_OUT_OPERATOR_FS_4: 292 + RECORDS_OUT_OPERATOR_FS_6: 184 + RECORDS_OUT_OPERATOR_GBY_2: 184 + RECORDS_OUT_OPERATOR_GBY_4: 184 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 184 + RECORDS_OUT_OPERATOR_SEL_1: 292 + RECORDS_OUT_OPERATOR_SEL_2: 292 + RECORDS_OUT_OPERATOR_SEL_5: 184 + RECORDS_OUT_OPERATOR_TS_0: 500 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -121,6 +169,22 @@ Stage-2 HIVE COUNTERS: RECORDS_OUT_1_default.src2: 84 RECORDS_OUT_2_default.src3: 105 RECORDS_OUT_INTERMEDIATE_Map_1: 121 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_3: 0 + RECORDS_OUT_OPERATOR_FIL_10: 105 + RECORDS_OUT_OPERATOR_FIL_9: 84 + RECORDS_OUT_OPERATOR_FS_4: 84 + RECORDS_OUT_OPERATOR_FS_6: 121 + RECORDS_OUT_OPERATOR_FS_8: 105 + RECORDS_OUT_OPERATOR_GBY_2: 121 + RECORDS_OUT_OPERATOR_GBY_4: 121 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 121 + RECORDS_OUT_OPERATOR_SEL_1: 189 + RECORDS_OUT_OPERATOR_SEL_2: 84 + RECORDS_OUT_OPERATOR_SEL_5: 121 + RECORDS_OUT_OPERATOR_SEL_6: 105 + RECORDS_OUT_OPERATOR_TS_0: 500 Stage-2 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -143,6 +207,22 @@ Stage-2 HIVE COUNTERS: RECORDS_OUT_1_default.src2: 84 RECORDS_OUT_2_default.src3: 208 RECORDS_OUT_INTERMEDIATE_Map_1: 184 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_3: 0 + RECORDS_OUT_OPERATOR_FIL_10: 208 + RECORDS_OUT_OPERATOR_FIL_9: 84 + RECORDS_OUT_OPERATOR_FS_4: 84 + RECORDS_OUT_OPERATOR_FS_6: 184 + RECORDS_OUT_OPERATOR_FS_8: 208 + RECORDS_OUT_OPERATOR_GBY_2: 184 + RECORDS_OUT_OPERATOR_GBY_4: 184 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 184 + RECORDS_OUT_OPERATOR_SEL_1: 292 + RECORDS_OUT_OPERATOR_SEL_2: 84 + RECORDS_OUT_OPERATOR_SEL_5: 184 + RECORDS_OUT_OPERATOR_SEL_6: 208 + RECORDS_OUT_OPERATOR_TS_0: 500 Stage-2 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -175,6 +255,21 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_1_default.src2: 189 RECORDS_OUT_INTERMEDIATE_Map_1: 57 RECORDS_OUT_INTERMEDIATE_Map_4: 64 + RECORDS_OUT_INTERMEDIATE_Reducer_3: 0 + RECORDS_OUT_OPERATOR_FIL_10: 84 + RECORDS_OUT_OPERATOR_FIL_11: 105 + RECORDS_OUT_OPERATOR_FS_6: 121 + RECORDS_OUT_OPERATOR_FS_9: 189 + RECORDS_OUT_OPERATOR_GBY_2: 121 + RECORDS_OUT_OPERATOR_GBY_4: 121 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 121 + RECORDS_OUT_OPERATOR_SEL_1: 189 + RECORDS_OUT_OPERATOR_SEL_2: 84 + RECORDS_OUT_OPERATOR_SEL_5: 226 + RECORDS_OUT_OPERATOR_SEL_7: 189 + RECORDS_OUT_OPERATOR_TS_0: 500 + RECORDS_OUT_OPERATOR_TS_3: 500 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 GROUPED_INPUT_SPLITS_Map_4: 1 @@ -203,6 +298,21 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_1_default.src2: 292 RECORDS_OUT_INTERMEDIATE_Map_1: 57 RECORDS_OUT_INTERMEDIATE_Map_4: 127 + RECORDS_OUT_INTERMEDIATE_Reducer_3: 0 + RECORDS_OUT_OPERATOR_FIL_10: 84 + RECORDS_OUT_OPERATOR_FIL_11: 208 + RECORDS_OUT_OPERATOR_FS_6: 184 + RECORDS_OUT_OPERATOR_FS_9: 292 + RECORDS_OUT_OPERATOR_GBY_2: 184 + RECORDS_OUT_OPERATOR_GBY_4: 184 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 184 + RECORDS_OUT_OPERATOR_SEL_1: 292 + RECORDS_OUT_OPERATOR_SEL_2: 84 + RECORDS_OUT_OPERATOR_SEL_5: 392 + RECORDS_OUT_OPERATOR_SEL_7: 292 + RECORDS_OUT_OPERATOR_TS_0: 500 + RECORDS_OUT_OPERATOR_TS_3: 500 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 GROUPED_INPUT_SPLITS_Map_4: 1 diff --git ql/src/test/results/clientpositive/llap/dp_counter_non_mm.q.out ql/src/test/results/clientpositive/llap/dp_counter_non_mm.q.out index 9f6426ccd1..28f376f8c4 100644 --- ql/src/test/results/clientpositive/llap/dp_counter_non_mm.q.out +++ ql/src/test/results/clientpositive/llap/dp_counter_non_mm.q.out @@ -23,6 +23,18 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 57 RECORDS_OUT_1_default.src2: 84 RECORDS_OUT_INTERMEDIATE_Map_1: 57 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_5: 84 + RECORDS_OUT_OPERATOR_FS_4: 84 + RECORDS_OUT_OPERATOR_FS_6: 57 + RECORDS_OUT_OPERATOR_GBY_2: 57 + RECORDS_OUT_OPERATOR_GBY_4: 57 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 57 + RECORDS_OUT_OPERATOR_SEL_1: 84 + RECORDS_OUT_OPERATOR_SEL_2: 84 + RECORDS_OUT_OPERATOR_SEL_5: 57 + RECORDS_OUT_OPERATOR_TS_0: 500 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -41,6 +53,18 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 121 RECORDS_OUT_1_default.src2: 189 RECORDS_OUT_INTERMEDIATE_Map_1: 121 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_5: 189 + RECORDS_OUT_OPERATOR_FS_4: 189 + RECORDS_OUT_OPERATOR_FS_6: 121 + RECORDS_OUT_OPERATOR_GBY_2: 121 + RECORDS_OUT_OPERATOR_GBY_4: 121 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 121 + RECORDS_OUT_OPERATOR_SEL_1: 189 + RECORDS_OUT_OPERATOR_SEL_2: 189 + RECORDS_OUT_OPERATOR_SEL_5: 121 + RECORDS_OUT_OPERATOR_TS_0: 500 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -67,6 +91,18 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 121 RECORDS_OUT_1_default.src2: 189 RECORDS_OUT_INTERMEDIATE_Map_1: 121 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_5: 189 + RECORDS_OUT_OPERATOR_FS_4: 189 + RECORDS_OUT_OPERATOR_FS_6: 121 + RECORDS_OUT_OPERATOR_GBY_2: 121 + RECORDS_OUT_OPERATOR_GBY_4: 121 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 121 + RECORDS_OUT_OPERATOR_SEL_1: 189 + RECORDS_OUT_OPERATOR_SEL_2: 189 + RECORDS_OUT_OPERATOR_SEL_5: 121 + RECORDS_OUT_OPERATOR_TS_0: 500 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -85,6 +121,18 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 184 RECORDS_OUT_1_default.src2: 292 RECORDS_OUT_INTERMEDIATE_Map_1: 184 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_5: 292 + RECORDS_OUT_OPERATOR_FS_4: 292 + RECORDS_OUT_OPERATOR_FS_6: 184 + RECORDS_OUT_OPERATOR_GBY_2: 184 + RECORDS_OUT_OPERATOR_GBY_4: 184 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 184 + RECORDS_OUT_OPERATOR_SEL_1: 292 + RECORDS_OUT_OPERATOR_SEL_2: 292 + RECORDS_OUT_OPERATOR_SEL_5: 184 + RECORDS_OUT_OPERATOR_TS_0: 500 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -121,6 +169,22 @@ Stage-2 HIVE COUNTERS: RECORDS_OUT_1_default.src2: 84 RECORDS_OUT_2_default.src3: 105 RECORDS_OUT_INTERMEDIATE_Map_1: 121 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_3: 0 + RECORDS_OUT_OPERATOR_FIL_10: 105 + RECORDS_OUT_OPERATOR_FIL_9: 84 + RECORDS_OUT_OPERATOR_FS_4: 84 + RECORDS_OUT_OPERATOR_FS_6: 121 + RECORDS_OUT_OPERATOR_FS_8: 105 + RECORDS_OUT_OPERATOR_GBY_2: 121 + RECORDS_OUT_OPERATOR_GBY_4: 121 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 121 + RECORDS_OUT_OPERATOR_SEL_1: 189 + RECORDS_OUT_OPERATOR_SEL_2: 84 + RECORDS_OUT_OPERATOR_SEL_5: 121 + RECORDS_OUT_OPERATOR_SEL_6: 105 + RECORDS_OUT_OPERATOR_TS_0: 500 Stage-2 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -143,6 +207,22 @@ Stage-2 HIVE COUNTERS: RECORDS_OUT_1_default.src2: 84 RECORDS_OUT_2_default.src3: 208 RECORDS_OUT_INTERMEDIATE_Map_1: 184 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_3: 0 + RECORDS_OUT_OPERATOR_FIL_10: 208 + RECORDS_OUT_OPERATOR_FIL_9: 84 + RECORDS_OUT_OPERATOR_FS_4: 84 + RECORDS_OUT_OPERATOR_FS_6: 184 + RECORDS_OUT_OPERATOR_FS_8: 208 + RECORDS_OUT_OPERATOR_GBY_2: 184 + RECORDS_OUT_OPERATOR_GBY_4: 184 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 184 + RECORDS_OUT_OPERATOR_SEL_1: 292 + RECORDS_OUT_OPERATOR_SEL_2: 84 + RECORDS_OUT_OPERATOR_SEL_5: 184 + RECORDS_OUT_OPERATOR_SEL_6: 208 + RECORDS_OUT_OPERATOR_TS_0: 500 Stage-2 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -175,6 +255,21 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_1_default.src2: 189 RECORDS_OUT_INTERMEDIATE_Map_1: 57 RECORDS_OUT_INTERMEDIATE_Map_4: 64 + RECORDS_OUT_INTERMEDIATE_Reducer_3: 0 + RECORDS_OUT_OPERATOR_FIL_10: 84 + RECORDS_OUT_OPERATOR_FIL_11: 105 + RECORDS_OUT_OPERATOR_FS_6: 121 + RECORDS_OUT_OPERATOR_FS_9: 189 + RECORDS_OUT_OPERATOR_GBY_2: 121 + RECORDS_OUT_OPERATOR_GBY_4: 121 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 121 + RECORDS_OUT_OPERATOR_SEL_1: 189 + RECORDS_OUT_OPERATOR_SEL_2: 84 + RECORDS_OUT_OPERATOR_SEL_5: 226 + RECORDS_OUT_OPERATOR_SEL_7: 189 + RECORDS_OUT_OPERATOR_TS_0: 500 + RECORDS_OUT_OPERATOR_TS_3: 500 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 GROUPED_INPUT_SPLITS_Map_4: 1 @@ -203,6 +298,21 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_1_default.src2: 292 RECORDS_OUT_INTERMEDIATE_Map_1: 57 RECORDS_OUT_INTERMEDIATE_Map_4: 127 + RECORDS_OUT_INTERMEDIATE_Reducer_3: 0 + RECORDS_OUT_OPERATOR_FIL_10: 84 + RECORDS_OUT_OPERATOR_FIL_11: 208 + RECORDS_OUT_OPERATOR_FS_6: 184 + RECORDS_OUT_OPERATOR_FS_9: 292 + RECORDS_OUT_OPERATOR_GBY_2: 184 + RECORDS_OUT_OPERATOR_GBY_4: 184 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 184 + RECORDS_OUT_OPERATOR_SEL_1: 292 + RECORDS_OUT_OPERATOR_SEL_2: 84 + RECORDS_OUT_OPERATOR_SEL_5: 392 + RECORDS_OUT_OPERATOR_SEL_7: 292 + RECORDS_OUT_OPERATOR_TS_0: 500 + RECORDS_OUT_OPERATOR_TS_3: 500 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 GROUPED_INPUT_SPLITS_Map_4: 1 diff --git ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out index aa2dcc7f68..b5497dac50 100644 --- ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out +++ ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out @@ -261,6 +261,13 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2100 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_9: 1 + RECORDS_OUT_OPERATOR_GBY_8: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_7: 2100 + RECORDS_OUT_OPERATOR_SEL_6: 2100 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 262144 ALLOCATED_USED_BYTES: 26 @@ -289,6 +296,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 RECORDS_OUT_0: 1 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 0 INPUT_DIRECTORIES_Map_1: 1 @@ -311,6 +321,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 8 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 8 + RECORDS_OUT_OPERATOR_SEL_9: 8 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 1048576 ALLOCATED_USED_BYTES: 2732 @@ -343,6 +361,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 22 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 22 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 22 + RECORDS_OUT_OPERATOR_SEL_9: 22 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -373,6 +399,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 16 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 16 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 16 + RECORDS_OUT_OPERATOR_SEL_9: 16 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -403,6 +437,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 18 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 18 + RECORDS_OUT_OPERATOR_SEL_9: 18 + RECORDS_OUT_OPERATOR_TS_0: 2000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -433,6 +475,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 1 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 1 + RECORDS_OUT_OPERATOR_SEL_9: 1 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -463,6 +513,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 32 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 32 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 32 + RECORDS_OUT_OPERATOR_SEL_9: 32 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -493,6 +551,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 32 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 32 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 32 + RECORDS_OUT_OPERATOR_SEL_9: 32 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -523,6 +589,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1697 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 1697 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 1697 + RECORDS_OUT_OPERATOR_SEL_9: 1697 + RECORDS_OUT_OPERATOR_TS_0: 2000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -553,6 +627,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 12 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 12 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 12 + RECORDS_OUT_OPERATOR_SEL_9: 12 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -583,6 +665,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1713 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 1713 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 1713 + RECORDS_OUT_OPERATOR_SEL_9: 1713 + RECORDS_OUT_OPERATOR_TS_0: 2000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -613,6 +703,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 6 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 6 + RECORDS_OUT_OPERATOR_SEL_9: 6 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -643,6 +741,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 50 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 50 + RECORDS_OUT_OPERATOR_FS_9: 1 + RECORDS_OUT_OPERATOR_GBY_8: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 50 + RECORDS_OUT_OPERATOR_SEL_2: 50 + RECORDS_OUT_OPERATOR_TS_0: 1100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -673,6 +779,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 318 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 318 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 318 + RECORDS_OUT_OPERATOR_SEL_9: 318 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -700,6 +814,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 RECORDS_OUT_0: 1 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 0 INPUT_DIRECTORIES_Map_1: 1 @@ -719,6 +836,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 RECORDS_OUT_0: 1 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 0 INPUT_DIRECTORIES_Map_1: 1 @@ -741,6 +861,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 0 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 0 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 0 + RECORDS_OUT_OPERATOR_SEL_9: 0 + RECORDS_OUT_OPERATOR_TS_0: 0 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 830 CACHE_MISS_BYTES: 0 @@ -768,6 +896,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 0 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 0 + RECORDS_OUT_OPERATOR_FS_9: 1 + RECORDS_OUT_OPERATOR_GBY_8: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 0 + RECORDS_OUT_OPERATOR_SEL_2: 0 + RECORDS_OUT_OPERATOR_TS_0: 0 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 830 CACHE_MISS_BYTES: 0 @@ -792,6 +928,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 RECORDS_OUT_0: 1 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 0 INPUT_DIRECTORIES_Map_1: 1 @@ -814,6 +953,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 2 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 2 + RECORDS_OUT_OPERATOR_SEL_9: 2 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 1310720 ALLOCATED_USED_BYTES: 13796 @@ -846,6 +993,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 6 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 6 + RECORDS_OUT_OPERATOR_SEL_9: 6 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -876,6 +1031,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 6 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 6 + RECORDS_OUT_OPERATOR_SEL_9: 6 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -903,6 +1066,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 RECORDS_OUT_0: 1 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 0 INPUT_DIRECTORIES_Map_1: 1 @@ -925,6 +1091,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2100 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 2100 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 2100 + RECORDS_OUT_OPERATOR_SEL_9: 2100 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -955,6 +1129,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 0 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 0 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 0 + RECORDS_OUT_OPERATOR_SEL_9: 0 + RECORDS_OUT_OPERATOR_TS_0: 0 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1735 CACHE_MISS_BYTES: 0 @@ -982,6 +1164,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 2 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 2 + RECORDS_OUT_OPERATOR_SEL_9: 2 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -1012,6 +1202,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 2 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 2 + RECORDS_OUT_OPERATOR_SEL_9: 2 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -1042,6 +1240,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 2 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 2 + RECORDS_OUT_OPERATOR_SEL_9: 2 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -1072,6 +1278,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 81 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 81 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 81 + RECORDS_OUT_OPERATOR_SEL_9: 81 + RECORDS_OUT_OPERATOR_TS_0: 2000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -1102,6 +1316,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 74 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 74 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 74 + RECORDS_OUT_OPERATOR_SEL_9: 74 + RECORDS_OUT_OPERATOR_TS_0: 2000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -1132,6 +1354,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 12 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 12 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 12 + RECORDS_OUT_OPERATOR_SEL_9: 12 + RECORDS_OUT_OPERATOR_TS_0: 2000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -1162,6 +1392,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 13 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 13 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 13 + RECORDS_OUT_OPERATOR_SEL_9: 13 + RECORDS_OUT_OPERATOR_TS_0: 2000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -1192,6 +1430,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 1 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 1 + RECORDS_OUT_OPERATOR_SEL_9: 1 + RECORDS_OUT_OPERATOR_TS_0: 100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -1222,6 +1468,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 7 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 7 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 7 + RECORDS_OUT_OPERATOR_SEL_9: 7 + RECORDS_OUT_OPERATOR_TS_0: 1100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -1252,6 +1506,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 0 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 0 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 0 + RECORDS_OUT_OPERATOR_SEL_9: 0 + RECORDS_OUT_OPERATOR_TS_0: 0 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1735 CACHE_MISS_BYTES: 0 @@ -1279,6 +1541,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 0 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 0 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 0 + RECORDS_OUT_OPERATOR_SEL_9: 0 + RECORDS_OUT_OPERATOR_TS_0: 0 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1735 CACHE_MISS_BYTES: 0 @@ -1306,6 +1576,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 0 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 0 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 0 + RECORDS_OUT_OPERATOR_SEL_9: 0 + RECORDS_OUT_OPERATOR_TS_0: 0 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1735 CACHE_MISS_BYTES: 0 @@ -1333,6 +1611,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 2 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 2 + RECORDS_OUT_OPERATOR_SEL_9: 2 + RECORDS_OUT_OPERATOR_TS_0: 100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -1363,6 +1649,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 6 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 6 + RECORDS_OUT_OPERATOR_SEL_9: 6 + RECORDS_OUT_OPERATOR_TS_0: 1100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -1393,6 +1687,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 2 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 2 + RECORDS_OUT_OPERATOR_SEL_9: 2 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 6770 CACHE_MISS_BYTES: 0 @@ -1423,6 +1725,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 2 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 2 + RECORDS_OUT_OPERATOR_SEL_9: 2 + RECORDS_OUT_OPERATOR_TS_0: 100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 6770 CACHE_MISS_BYTES: 0 diff --git ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out index eb54a81d1a..dc5d48c385 100644 --- ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out +++ ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out @@ -261,6 +261,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2094 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 2094 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 2094 + RECORDS_OUT_OPERATOR_SEL_9: 2094 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 1310720 ALLOCATED_USED_BYTES: 2758 @@ -293,6 +301,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2094 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_8: 2094 + RECORDS_OUT_OPERATOR_FS_12: 1 + RECORDS_OUT_OPERATOR_GBY_11: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_10: 2094 + RECORDS_OUT_OPERATOR_SEL_9: 2094 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 diff --git ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out index 590437c708..411fa72df2 100644 --- ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out +++ ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out @@ -214,6 +214,13 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2100 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_5: 1 + RECORDS_OUT_OPERATOR_GBY_3: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_2: 2100 + RECORDS_OUT_OPERATOR_SEL_1: 2100 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 262144 ALLOCATED_USED_BYTES: 26 @@ -242,6 +249,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 RECORDS_OUT_0: 1 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 0 INPUT_DIRECTORIES_Map_1: 1 @@ -264,6 +274,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 8 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 8 + RECORDS_OUT_OPERATOR_SEL_2: 8 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 1048576 ALLOCATED_USED_BYTES: 2732 @@ -296,6 +314,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 22 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 22 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 22 + RECORDS_OUT_OPERATOR_SEL_2: 22 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -326,6 +352,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 16 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 16 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 16 + RECORDS_OUT_OPERATOR_SEL_2: 16 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -356,6 +390,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 18 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 18 + RECORDS_OUT_OPERATOR_SEL_2: 18 + RECORDS_OUT_OPERATOR_TS_0: 2000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -386,6 +428,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 1 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 1 + RECORDS_OUT_OPERATOR_SEL_2: 1 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -416,6 +466,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 32 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 32 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 32 + RECORDS_OUT_OPERATOR_SEL_2: 32 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -446,6 +504,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 32 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 32 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 32 + RECORDS_OUT_OPERATOR_SEL_2: 32 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -476,6 +542,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1697 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 1697 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 1697 + RECORDS_OUT_OPERATOR_SEL_2: 1697 + RECORDS_OUT_OPERATOR_TS_0: 2000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -506,6 +580,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 12 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 12 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 12 + RECORDS_OUT_OPERATOR_SEL_2: 12 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -536,6 +618,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1713 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 1713 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 1713 + RECORDS_OUT_OPERATOR_SEL_2: 1713 + RECORDS_OUT_OPERATOR_TS_0: 2000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -566,6 +656,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 6 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 6 + RECORDS_OUT_OPERATOR_SEL_2: 6 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -596,6 +694,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 50 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 50 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 50 + RECORDS_OUT_OPERATOR_SEL_2: 50 + RECORDS_OUT_OPERATOR_TS_0: 1100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -626,6 +732,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 318 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 318 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 318 + RECORDS_OUT_OPERATOR_SEL_2: 318 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -653,6 +767,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 RECORDS_OUT_0: 1 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 0 INPUT_DIRECTORIES_Map_1: 1 @@ -672,6 +789,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 RECORDS_OUT_0: 1 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 0 INPUT_DIRECTORIES_Map_1: 1 @@ -694,6 +814,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 0 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 0 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 0 + RECORDS_OUT_OPERATOR_SEL_2: 0 + RECORDS_OUT_OPERATOR_TS_0: 0 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 830 CACHE_MISS_BYTES: 0 @@ -721,6 +849,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 0 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 0 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 0 + RECORDS_OUT_OPERATOR_SEL_2: 0 + RECORDS_OUT_OPERATOR_TS_0: 0 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 830 CACHE_MISS_BYTES: 0 @@ -745,6 +881,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 RECORDS_OUT_0: 1 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 0 INPUT_DIRECTORIES_Map_1: 1 @@ -767,6 +906,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 2 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 2 + RECORDS_OUT_OPERATOR_SEL_2: 2 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 1310720 ALLOCATED_USED_BYTES: 13796 @@ -799,6 +946,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 6 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 6 + RECORDS_OUT_OPERATOR_SEL_2: 6 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -829,6 +984,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 6 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 6 + RECORDS_OUT_OPERATOR_SEL_2: 6 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -856,6 +1019,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 RECORDS_OUT_0: 1 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 0 INPUT_DIRECTORIES_Map_1: 1 @@ -878,6 +1044,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2100 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 2100 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 2100 + RECORDS_OUT_OPERATOR_SEL_2: 2100 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -908,6 +1082,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 0 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 0 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 0 + RECORDS_OUT_OPERATOR_SEL_2: 0 + RECORDS_OUT_OPERATOR_TS_0: 0 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1735 CACHE_MISS_BYTES: 0 @@ -935,6 +1117,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 2 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 2 + RECORDS_OUT_OPERATOR_SEL_2: 2 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -965,6 +1155,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 2 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 2 + RECORDS_OUT_OPERATOR_SEL_2: 2 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -995,6 +1193,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 2 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 2 + RECORDS_OUT_OPERATOR_SEL_2: 2 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -1025,6 +1231,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 81 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 81 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 81 + RECORDS_OUT_OPERATOR_SEL_2: 81 + RECORDS_OUT_OPERATOR_TS_0: 2000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -1055,6 +1269,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 74 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 74 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 74 + RECORDS_OUT_OPERATOR_SEL_2: 74 + RECORDS_OUT_OPERATOR_TS_0: 2000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -1085,6 +1307,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 12 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 12 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 12 + RECORDS_OUT_OPERATOR_SEL_2: 12 + RECORDS_OUT_OPERATOR_TS_0: 2000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -1115,6 +1345,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 13 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 13 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 13 + RECORDS_OUT_OPERATOR_SEL_2: 13 + RECORDS_OUT_OPERATOR_TS_0: 2000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -1145,6 +1383,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 1 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 1 + RECORDS_OUT_OPERATOR_SEL_2: 1 + RECORDS_OUT_OPERATOR_TS_0: 100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -1175,6 +1421,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 7 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 7 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 7 + RECORDS_OUT_OPERATOR_SEL_2: 7 + RECORDS_OUT_OPERATOR_TS_0: 1100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -1205,6 +1459,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 0 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 0 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 0 + RECORDS_OUT_OPERATOR_SEL_2: 0 + RECORDS_OUT_OPERATOR_TS_0: 0 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1735 CACHE_MISS_BYTES: 0 @@ -1232,6 +1494,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 0 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 0 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 0 + RECORDS_OUT_OPERATOR_SEL_2: 0 + RECORDS_OUT_OPERATOR_TS_0: 0 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1735 CACHE_MISS_BYTES: 0 @@ -1259,6 +1529,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 0 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 0 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 0 + RECORDS_OUT_OPERATOR_SEL_2: 0 + RECORDS_OUT_OPERATOR_TS_0: 0 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1735 CACHE_MISS_BYTES: 0 @@ -1286,6 +1564,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 2 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 2 + RECORDS_OUT_OPERATOR_SEL_2: 2 + RECORDS_OUT_OPERATOR_TS_0: 100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -1316,6 +1602,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 6 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 6 + RECORDS_OUT_OPERATOR_SEL_2: 6 + RECORDS_OUT_OPERATOR_TS_0: 1100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -1346,6 +1640,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 2 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 2 + RECORDS_OUT_OPERATOR_SEL_2: 2 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 6770 CACHE_MISS_BYTES: 0 @@ -1376,6 +1678,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 2 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 2 + RECORDS_OUT_OPERATOR_SEL_2: 2 + RECORDS_OUT_OPERATOR_TS_0: 100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 6770 CACHE_MISS_BYTES: 0 @@ -1406,6 +1716,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 2 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 2 + RECORDS_OUT_OPERATOR_SEL_2: 2 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 524288 ALLOCATED_USED_BYTES: 8527 @@ -1438,6 +1756,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 2 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 2 + RECORDS_OUT_OPERATOR_SEL_2: 2 + RECORDS_OUT_OPERATOR_TS_0: 100 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 262144 ALLOCATED_USED_BYTES: 2376 @@ -1470,6 +1796,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 2 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 2 + RECORDS_OUT_OPERATOR_SEL_2: 2 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4936 CACHE_MISS_BYTES: 0 @@ -1500,6 +1834,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 2 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 2 + RECORDS_OUT_OPERATOR_SEL_2: 2 + RECORDS_OUT_OPERATOR_TS_0: 100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 6687 CACHE_MISS_BYTES: 0 @@ -1530,6 +1872,12 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 98 RECORDS_OUT_1_default.orc_ppd_1: 2 + RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_OPERATOR_FIL_4: 2 + RECORDS_OUT_OPERATOR_FS_3: 2 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_SEL_2: 2 + RECORDS_OUT_OPERATOR_TS_0: 98 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 2359296 ALLOCATED_USED_BYTES: 44166 @@ -1561,6 +1909,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 2 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 2 + RECORDS_OUT_OPERATOR_SEL_2: 2 + RECORDS_OUT_OPERATOR_TS_0: 2 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 786432 ALLOCATED_USED_BYTES: 65 @@ -1593,6 +1949,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 2 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 2 + RECORDS_OUT_OPERATOR_SEL_2: 2 + RECORDS_OUT_OPERATOR_TS_0: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 62 CACHE_MISS_BYTES: 0 @@ -1623,6 +1987,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 2 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 2 + RECORDS_OUT_OPERATOR_SEL_2: 2 + RECORDS_OUT_OPERATOR_TS_0: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 62 CACHE_MISS_BYTES: 0 @@ -1653,6 +2025,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 2 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 2 + RECORDS_OUT_OPERATOR_SEL_2: 2 + RECORDS_OUT_OPERATOR_TS_0: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 62 CACHE_MISS_BYTES: 0 diff --git ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out index c7c8993d0e..e87cd287ac 100644 --- ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out +++ ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out @@ -214,6 +214,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 0 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 0 + RECORDS_OUT_OPERATOR_SEL_2: 0 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 1048576 ALLOCATED_USED_BYTES: 382 @@ -243,6 +251,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 RECORDS_OUT_0: 1 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 0 INPUT_DIRECTORIES_Map_1: 1 @@ -265,6 +276,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 8 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 8 + RECORDS_OUT_OPERATOR_SEL_2: 8 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 359 CACHE_MISS_BYTES: 0 @@ -295,6 +314,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 8 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 8 + RECORDS_OUT_OPERATOR_SEL_2: 8 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 262144 ALLOCATED_USED_BYTES: 2376 @@ -327,6 +354,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 18 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 18 + RECORDS_OUT_OPERATOR_SEL_2: 18 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 359 CACHE_MISS_BYTES: 0 @@ -357,6 +392,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 18 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 18 + RECORDS_OUT_OPERATOR_SEL_2: 18 + RECORDS_OUT_OPERATOR_TS_0: 2000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -391,6 +434,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 0 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 0 + RECORDS_OUT_OPERATOR_SEL_2: 0 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 359 CACHE_MISS_BYTES: 0 @@ -418,6 +469,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 RECORDS_OUT_0: 1 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 0 INPUT_DIRECTORIES_Map_1: 1 @@ -440,6 +494,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 8 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 8 + RECORDS_OUT_OPERATOR_SEL_2: 8 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 359 CACHE_MISS_BYTES: 0 @@ -470,6 +532,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 8 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 8 + RECORDS_OUT_OPERATOR_SEL_2: 8 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -500,6 +570,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 18 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 18 + RECORDS_OUT_OPERATOR_SEL_2: 18 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 359 CACHE_MISS_BYTES: 0 @@ -530,6 +608,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 18 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 18 + RECORDS_OUT_OPERATOR_SEL_2: 18 + RECORDS_OUT_OPERATOR_TS_0: 2000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -564,6 +650,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 0 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 0 + RECORDS_OUT_OPERATOR_SEL_2: 0 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 359 CACHE_MISS_BYTES: 0 @@ -591,6 +685,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 RECORDS_OUT_0: 1 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 0 INPUT_DIRECTORIES_Map_1: 1 @@ -613,6 +710,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 8 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 8 + RECORDS_OUT_OPERATOR_SEL_2: 8 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 359 CACHE_MISS_BYTES: 0 @@ -643,6 +748,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 8 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 8 + RECORDS_OUT_OPERATOR_SEL_2: 8 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -673,6 +786,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 18 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 18 + RECORDS_OUT_OPERATOR_SEL_2: 18 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 359 CACHE_MISS_BYTES: 0 @@ -703,6 +824,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 18 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 18 + RECORDS_OUT_OPERATOR_SEL_2: 18 + RECORDS_OUT_OPERATOR_TS_0: 2000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -737,6 +866,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 0 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 0 + RECORDS_OUT_OPERATOR_SEL_2: 0 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 359 CACHE_MISS_BYTES: 0 @@ -764,6 +901,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 RECORDS_OUT_0: 1 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 0 INPUT_DIRECTORIES_Map_1: 1 @@ -786,6 +926,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 8 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 8 + RECORDS_OUT_OPERATOR_SEL_2: 8 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 359 CACHE_MISS_BYTES: 0 @@ -816,6 +964,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 8 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 8 + RECORDS_OUT_OPERATOR_SEL_2: 8 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -846,6 +1002,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 18 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 18 + RECORDS_OUT_OPERATOR_SEL_2: 18 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 359 CACHE_MISS_BYTES: 0 @@ -876,6 +1040,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 18 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 18 + RECORDS_OUT_OPERATOR_SEL_2: 18 + RECORDS_OUT_OPERATOR_TS_0: 2000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -910,6 +1082,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1566 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 1566 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 1566 + RECORDS_OUT_OPERATOR_SEL_2: 1566 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 Stage-1 INPUT COUNTERS: @@ -934,6 +1114,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1566 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 1566 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 1566 + RECORDS_OUT_OPERATOR_SEL_2: 1566 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 Stage-1 INPUT COUNTERS: @@ -958,6 +1146,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 8 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 8 + RECORDS_OUT_OPERATOR_SEL_2: 8 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 Stage-1 INPUT COUNTERS: @@ -982,6 +1178,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 8 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 8 + RECORDS_OUT_OPERATOR_SEL_2: 8 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 Stage-1 INPUT COUNTERS: @@ -1006,6 +1210,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 18 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 18 + RECORDS_OUT_OPERATOR_SEL_2: 18 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 Stage-1 INPUT COUNTERS: @@ -1030,6 +1242,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 18 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 18 + RECORDS_OUT_OPERATOR_SEL_2: 18 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 Stage-1 INPUT COUNTERS: @@ -1054,6 +1274,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 2 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 2 + RECORDS_OUT_OPERATOR_SEL_2: 2 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 524288 ALLOCATED_USED_BYTES: 8527 @@ -1086,6 +1314,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 2 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 2 + RECORDS_OUT_OPERATOR_SEL_2: 2 + RECORDS_OUT_OPERATOR_TS_0: 100 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 262144 ALLOCATED_USED_BYTES: 2376 @@ -1122,6 +1358,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 2 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 2 + RECORDS_OUT_OPERATOR_SEL_2: 2 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 Stage-1 INPUT COUNTERS: @@ -1146,6 +1390,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 2 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 2 + RECORDS_OUT_OPERATOR_SEL_2: 2 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 Stage-1 INPUT COUNTERS: @@ -1174,6 +1426,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 2 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 2 + RECORDS_OUT_OPERATOR_SEL_2: 2 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 Stage-1 INPUT COUNTERS: @@ -1198,6 +1458,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 2 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 2 + RECORDS_OUT_OPERATOR_SEL_2: 2 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 Stage-1 INPUT COUNTERS: @@ -1222,6 +1490,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 6 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 6 + RECORDS_OUT_OPERATOR_SEL_2: 6 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 1048576 ALLOCATED_USED_BYTES: 11420 @@ -1254,6 +1530,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 6 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 6 + RECORDS_OUT_OPERATOR_SEL_2: 6 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 262144 ALLOCATED_USED_BYTES: 2376 @@ -1290,6 +1574,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 6 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 6 + RECORDS_OUT_OPERATOR_SEL_2: 6 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 Stage-1 INPUT COUNTERS: @@ -1314,6 +1606,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 6 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 6 + RECORDS_OUT_OPERATOR_SEL_2: 6 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 Stage-1 INPUT COUNTERS: @@ -1342,6 +1642,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 6 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 6 + RECORDS_OUT_OPERATOR_SEL_2: 6 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4123 CACHE_MISS_BYTES: 0 @@ -1372,6 +1680,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 6 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 6 + RECORDS_OUT_OPERATOR_SEL_2: 6 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -1406,6 +1722,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 6 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 6 + RECORDS_OUT_OPERATOR_SEL_2: 6 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 Stage-1 INPUT COUNTERS: @@ -1430,6 +1754,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 6 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 6 + RECORDS_OUT_OPERATOR_SEL_2: 6 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 Stage-1 INPUT COUNTERS: @@ -1458,6 +1790,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 6 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 6 + RECORDS_OUT_OPERATOR_SEL_2: 6 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4123 CACHE_MISS_BYTES: 0 @@ -1488,6 +1828,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 6 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 6 + RECORDS_OUT_OPERATOR_SEL_2: 6 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 5715 CACHE_MISS_BYTES: 0 @@ -1522,6 +1870,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 4 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 4 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 4 + RECORDS_OUT_OPERATOR_SEL_2: 4 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 786432 ALLOCATED_USED_BYTES: 4293 @@ -1554,6 +1910,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 4 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 4 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 4 + RECORDS_OUT_OPERATOR_SEL_2: 4 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 Stage-1 INPUT COUNTERS: @@ -1578,6 +1942,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 4 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 4 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 4 + RECORDS_OUT_OPERATOR_SEL_2: 4 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 262144 ALLOCATED_USED_BYTES: 2376 @@ -1610,6 +1982,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 4 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 4 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 4 + RECORDS_OUT_OPERATOR_SEL_2: 4 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 Stage-1 INPUT COUNTERS: diff --git ql/src/test/results/clientpositive/llap/retry_failure.q.out ql/src/test/results/clientpositive/llap/retry_failure.q.out new file mode 100644 index 0000000000..5f91ad1624 --- /dev/null +++ ql/src/test/results/clientpositive/llap/retry_failure.q.out @@ -0,0 +1,43 @@ +PREHOOK: query: create table tx(a int,f string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tx +POSTHOOK: query: create table tx(a int,f string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tx +PREHOOK: query: insert into tx values (1,'non_existent_file') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@tx +POSTHOOK: query: insert into tx values (1,'non_existent_file') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@tx +POSTHOOK: Lineage: tx.a SCRIPT [] +POSTHOOK: Lineage: tx.f SCRIPT [] +PREHOOK: query: select assert_true(1 > a) from tx group by a +PREHOOK: type: QUERY +PREHOOK: Input: default@tx +#### A masked pattern was here #### +Status: Failed +Vertex failed, vertexName=Reducer 2, vertexId=vertex_#ID#, diagnostics=[Task failed, taskId=task_#ID#, diagnostics=[TaskAttempt 0 failed, info=[Error: Error while running task ( failure ) : attempt_#ID#:java.lang.RuntimeException: java.lang.RuntimeException: Hive Runtime Error while closing operators: org.apache.hadoop.hive.ql.metadata.HiveException: org.apache.hadoop.hive.ql.metadata.HiveException: ASSERT_TRUE(): assertion failed. +#### A masked pattern was here #### +], TaskAttempt 1 failed, info=[Error: Error while running task ( failure ) : attempt_#ID#:java.lang.RuntimeException: java.lang.RuntimeException: Hive Runtime Error while closing operators: org.apache.hadoop.hive.ql.metadata.HiveException: org.apache.hadoop.hive.ql.metadata.HiveException: ASSERT_TRUE(): assertion failed. +#### A masked pattern was here #### +]], Vertex did not succeed due to OWN_TASK_FAILURE, failedTasks:1 killedTasks:0, Vertex vertex_#ID# [Reducer 2] killed/failed due to:OWN_TASK_FAILURE] +DAG did not succeed due to VERTEX_FAILURE. failedVertices:1 killedVertices:0 +FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.tez.TezTask. Vertex failed, vertexName=Reducer 2, vertexId=vertex_#ID#, diagnostics=[Task failed, taskId=task_#ID#, diagnostics=[TaskAttempt 0 failed, info=[Error: Error while running task ( failure ) : attempt_#ID#:java.lang.RuntimeException: java.lang.RuntimeException: Hive Runtime Error while closing operators: org.apache.hadoop.hive.ql.metadata.HiveException: org.apache.hadoop.hive.ql.metadata.HiveException: ASSERT_TRUE(): assertion failed. +#### A masked pattern was here #### +], TaskAttempt 1 failed, info=[Error: Error while running task ( failure ) : attempt_#ID#:java.lang.RuntimeException: java.lang.RuntimeException: Hive Runtime Error while closing operators: org.apache.hadoop.hive.ql.metadata.HiveException: org.apache.hadoop.hive.ql.metadata.HiveException: ASSERT_TRUE(): assertion failed. +#### A masked pattern was here #### +]], Vertex did not succeed due to OWN_TASK_FAILURE, failedTasks:1 killedTasks:0, Vertex vertex_#ID# [Reducer 2] killed/failed due to:OWN_TASK_FAILURE]DAG did not succeed due to VERTEX_FAILURE. failedVertices:1 killedVertices:0 +PREHOOK: query: select assert_true(2 > a) from tx group by a +PREHOOK: type: QUERY +PREHOOK: Input: default@tx +#### A masked pattern was here #### +POSTHOOK: query: select assert_true(2 > a) from tx group by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tx +#### A masked pattern was here #### +NULL diff --git ql/src/test/results/clientpositive/llap/retry_failure_oom.q.out ql/src/test/results/clientpositive/llap/retry_failure_oom.q.out new file mode 100644 index 0000000000..348384a692 --- /dev/null +++ ql/src/test/results/clientpositive/llap/retry_failure_oom.q.out @@ -0,0 +1,43 @@ +PREHOOK: query: create table tx(a int,f string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tx +POSTHOOK: query: create table tx(a int,f string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tx +PREHOOK: query: insert into tx values (1,'non_existent_file') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@tx +POSTHOOK: query: insert into tx values (1,'non_existent_file') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@tx +POSTHOOK: Lineage: tx.a SCRIPT [] +POSTHOOK: Lineage: tx.f SCRIPT [] +PREHOOK: query: select assert_true_oom(1 > a) from tx group by a +PREHOOK: type: QUERY +PREHOOK: Input: default@tx +#### A masked pattern was here #### +Status: Failed +Vertex failed, vertexName=Reducer 2, vertexId=vertex_#ID#, diagnostics=[Task failed, taskId=task_#ID#, diagnostics=[TaskAttempt 0 failed, info=[Error: Error while running task ( failure ) : attempt_#ID#:java.lang.RuntimeException: org.apache.hadoop.hive.ql.exec.mapjoin.MapJoinMemoryExhaustionError: assert_true_oom: assertation failed; Simulated OOM +#### A masked pattern was here #### +], TaskAttempt 1 failed, info=[Error: Error while running task ( failure ) : attempt_#ID#:java.lang.RuntimeException: org.apache.hadoop.hive.ql.exec.mapjoin.MapJoinMemoryExhaustionError: assert_true_oom: assertation failed; Simulated OOM +#### A masked pattern was here #### +]], Vertex did not succeed due to OWN_TASK_FAILURE, failedTasks:1 killedTasks:0, Vertex vertex_#ID# [Reducer 2] killed/failed due to:OWN_TASK_FAILURE] +DAG did not succeed due to VERTEX_FAILURE. failedVertices:1 killedVertices:0 +FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.tez.TezTask. Vertex failed, vertexName=Reducer 2, vertexId=vertex_#ID#, diagnostics=[Task failed, taskId=task_#ID#, diagnostics=[TaskAttempt 0 failed, info=[Error: Error while running task ( failure ) : attempt_#ID#:java.lang.RuntimeException: org.apache.hadoop.hive.ql.exec.mapjoin.MapJoinMemoryExhaustionError: assert_true_oom: assertation failed; Simulated OOM +#### A masked pattern was here #### +], TaskAttempt 1 failed, info=[Error: Error while running task ( failure ) : attempt_#ID#:java.lang.RuntimeException: org.apache.hadoop.hive.ql.exec.mapjoin.MapJoinMemoryExhaustionError: assert_true_oom: assertation failed; Simulated OOM +#### A masked pattern was here #### +]], Vertex did not succeed due to OWN_TASK_FAILURE, failedTasks:1 killedTasks:0, Vertex vertex_#ID# [Reducer 2] killed/failed due to:OWN_TASK_FAILURE]DAG did not succeed due to VERTEX_FAILURE. failedVertices:1 killedVertices:0 +PREHOOK: query: select assert_true_oom(2 > a) from tx group by a +PREHOOK: type: QUERY +PREHOOK: Input: default@tx +#### A masked pattern was here #### +POSTHOOK: query: select assert_true_oom(2 > a) from tx group by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tx +#### A masked pattern was here #### +NULL diff --git ql/src/test/results/clientpositive/llap/retry_failure_stat_changes.q.out ql/src/test/results/clientpositive/llap/retry_failure_stat_changes.q.out new file mode 100644 index 0000000000..9b58ce0e5d --- /dev/null +++ ql/src/test/results/clientpositive/llap/retry_failure_stat_changes.q.out @@ -0,0 +1,280 @@ +PREHOOK: query: create table tx(a int,u int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tx +POSTHOOK: query: create table tx(a int,u int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tx +PREHOOK: query: insert into tx values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(10,10) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@tx +POSTHOOK: query: insert into tx values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(10,10) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@tx +POSTHOOK: Lineage: tx.a SCRIPT [] +POSTHOOK: Lineage: tx.u SCRIPT [] +PREHOOK: query: create table px(a int,p int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@px +POSTHOOK: query: create table px(a int,p int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@px +PREHOOK: query: insert into px values (2,2),(3,3),(5,5),(7,7),(11,11) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@px +POSTHOOK: query: insert into px values (2,2),(3,3),(5,5),(7,7),(11,11) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@px +POSTHOOK: Lineage: px.a SCRIPT [] +POSTHOOK: Lineage: px.p SCRIPT [] +PREHOOK: query: explain REOPTIMIZATION +select sum(u*p) from tx join px on (u=p) where u<10 and p>2 +PREHOOK: type: QUERY +PREHOOK: Input: default@px +PREHOOK: Input: default@tx +#### A masked pattern was here #### +POSTHOOK: query: explain REOPTIMIZATION +select sum(u*p) from tx join px on (u=p) where u<10 and p>2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@px +POSTHOOK: Input: default@tx +#### A masked pattern was here #### +PREHOOK: query: explain REOPTIMIZATION +select sum(u*p) from tx join px on (u=p) where u<10 and p>2 +PREHOOK: type: QUERY +POSTHOOK: query: explain REOPTIMIZATION +select sum(u*p) from tx join px on (u=p) where u<10 and p>2 +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 llap + File Output Operator [FS_15] + Group By Operator [GBY_13] (rows=1 width=8) + Output:["_col0"],aggregations:["sum(VALUE._col0)"] + <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap + PARTITION_ONLY_SHUFFLE [RS_12] + Group By Operator [GBY_11] (rows=1 width=8) + Output:["_col0"],aggregations:["sum(_col0)"] + Select Operator [SEL_9] (rows=15 width=8) + Output:["_col0"] + Merge Join Operator [MERGEJOIN_20] (rows=15 width=8) + Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0","_col1"] + <-Map 1 [SIMPLE_EDGE] llap + SHUFFLE [RS_6] + PartitionCols:_col0 + Select Operator [SEL_2] (runtime: rows=5 width=4) + Output:["_col0"] + Filter Operator [FIL_18] (runtime: rows=5 width=4) + predicate:((u < 10) and (u > 2)) + TableScan [TS_0] (runtime: rows=8 width=4) + default@tx,tx,Tbl:COMPLETE,Col:COMPLETE,Output:["u"] + <-Map 4 [SIMPLE_EDGE] llap + SHUFFLE [RS_7] + PartitionCols:_col0 + Select Operator [SEL_5] (runtime: rows=3 width=4) + Output:["_col0"] + Filter Operator [FIL_19] (runtime: rows=3 width=4) + predicate:((p < 10) and (p > 2)) + TableScan [TS_3] (runtime: rows=5 width=4) + default@px,px,Tbl:COMPLETE,Col:COMPLETE,Output:["p"] + +PREHOOK: query: select sum(u*p) from tx join px on (u=p) where u<10 and p>2 +PREHOOK: type: QUERY +PREHOOK: Input: default@px +PREHOOK: Input: default@tx +#### A masked pattern was here #### +POSTHOOK: query: select sum(u*p) from tx join px on (u=p) where u<10 and p>2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@px +POSTHOOK: Input: default@tx +#### A masked pattern was here #### +PREHOOK: query: explain analyze +select sum(u*p) from tx join px on (u=p) where u<10 and p>2 +PREHOOK: type: QUERY +POSTHOOK: query: explain analyze +select sum(u*p) from tx join px on (u=p) where u<10 and p>2 +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 llap + File Output Operator [FS_15] + Group By Operator [GBY_13] (rows=1/1 width=8) + Output:["_col0"],aggregations:["sum(VALUE._col0)"] + <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap + PARTITION_ONLY_SHUFFLE [RS_12] + Group By Operator [GBY_11] (rows=1/1 width=8) + Output:["_col0"],aggregations:["sum(_col0)"] + Select Operator [SEL_9] (rows=1/3 width=8) + Output:["_col0"] + Merge Join Operator [MERGEJOIN_20] (rows=1/3 width=8) + Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0","_col1"] + <-Map 1 [SIMPLE_EDGE] llap + SHUFFLE [RS_6] + PartitionCols:_col0 + Select Operator [SEL_2] (rows=1/5 width=4) + Output:["_col0"] + Filter Operator [FIL_18] (rows=1/5 width=4) + predicate:((u < 10) and (u > 2)) + TableScan [TS_0] (rows=8/8 width=4) + default@tx,tx,Tbl:COMPLETE,Col:COMPLETE,Output:["u"] + <-Map 4 [SIMPLE_EDGE] llap + SHUFFLE [RS_7] + PartitionCols:_col0 + Select Operator [SEL_5] (rows=1/3 width=4) + Output:["_col0"] + Filter Operator [FIL_19] (rows=1/3 width=4) + predicate:((p < 10) and (p > 2)) + TableScan [TS_3] (rows=5/5 width=4) + default@px,px,Tbl:COMPLETE,Col:COMPLETE,Output:["p"] + +PREHOOK: query: select sum(u*p) from tx join px on (u=p) where u<10 and p>2 +PREHOOK: type: QUERY +PREHOOK: Input: default@px +PREHOOK: Input: default@tx +#### A masked pattern was here #### +POSTHOOK: query: select sum(u*p) from tx join px on (u=p) where u<10 and p>2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@px +POSTHOOK: Input: default@tx +#### A masked pattern was here #### +PREHOOK: query: explain analyze +select sum(u*p) from tx join px on (u=p) where u<10 and p>2 +PREHOOK: type: QUERY +POSTHOOK: query: explain analyze +select sum(u*p) from tx join px on (u=p) where u<10 and p>2 +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Map 1 <- Map 3 (BROADCAST_EDGE) +Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 llap + File Output Operator [FS_15] + Group By Operator [GBY_13] (rows=1/1 width=8) + Output:["_col0"],aggregations:["sum(VALUE._col0)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] llap + PARTITION_ONLY_SHUFFLE [RS_12] + Group By Operator [GBY_11] (rows=1/1 width=8) + Output:["_col0"],aggregations:["sum(_col0)"] + Select Operator [SEL_9] (rows=1/3 width=8) + Output:["_col0"] + Map Join Operator [MAPJOIN_20] (rows=1/3 width=8) + Conds:SEL_2._col0=RS_7._col0(Inner),Output:["_col0","_col1"] + <-Map 3 [BROADCAST_EDGE] llap + BROADCAST [RS_7] + PartitionCols:_col0 + Select Operator [SEL_5] (rows=1/3 width=4) + Output:["_col0"] + Filter Operator [FIL_19] (rows=1/3 width=4) + predicate:((p < 10) and (p > 2)) + TableScan [TS_3] (rows=5/5 width=4) + default@px,px,Tbl:COMPLETE,Col:COMPLETE,Output:["p"] + <-Select Operator [SEL_2] (rows=1/5 width=4) + Output:["_col0"] + Filter Operator [FIL_18] (rows=1/5 width=4) + predicate:((u < 10) and (u > 2)) + TableScan [TS_0] (rows=8/8 width=4) + default@tx,tx,Tbl:COMPLETE,Col:COMPLETE,Output:["u"] + +PREHOOK: query: explain +select assert_true_oom(1 > sum(u*p)) from tx join px on (tx.a=px.a) where u<10 and p>2 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select assert_true_oom(1 > sum(u*p)) from tx join px on (tx.a=px.a) where u<10 and p>2 +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Map 2 <- Map 1 (BROADCAST_EDGE) +Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 3 llap + File Output Operator [FS_15] + Select Operator [SEL_14] (rows=1 width=4) + Output:["_col0"] + Group By Operator [GBY_13] (rows=1 width=8) + Output:["_col0"],aggregations:["sum(VALUE._col0)"] + <-Map 2 [CUSTOM_SIMPLE_EDGE] llap + PARTITION_ONLY_SHUFFLE [RS_12] + Group By Operator [GBY_11] (rows=1 width=8) + Output:["_col0"],aggregations:["sum(_col0)"] + Select Operator [SEL_9] (rows=2 width=8) + Output:["_col0"] + Map Join Operator [MAPJOIN_20] (rows=2 width=8) + Conds:RS_6._col0=SEL_5._col0(Inner),Output:["_col1","_col3"] + <-Map 1 [BROADCAST_EDGE] llap + BROADCAST [RS_6] + PartitionCols:_col0 + Select Operator [SEL_2] (rows=2 width=8) + Output:["_col0","_col1"] + Filter Operator [FIL_18] (rows=2 width=8) + predicate:((u < 10) and a is not null) + TableScan [TS_0] (rows=8 width=8) + default@tx,tx,Tbl:COMPLETE,Col:COMPLETE,Output:["a","u"] + <-Select Operator [SEL_5] (rows=5 width=8) + Output:["_col0","_col1"] + Filter Operator [FIL_19] (rows=5 width=8) + predicate:((p > 2) and a is not null) + TableScan [TS_3] (rows=5 width=8) + default@px,px,Tbl:COMPLETE,Col:COMPLETE,Output:["a","p"] + +PREHOOK: query: select assert_true_oom(1 > sum(u*p)) from tx join px on (tx.a=px.a) where u<10 and p>2 +PREHOOK: type: QUERY +PREHOOK: Input: default@px +PREHOOK: Input: default@tx +#### A masked pattern was here #### +Status: Failed +Vertex failed, vertexName=Reducer 3, vertexId=vertex_#ID#, diagnostics=[Task failed, taskId=task_#ID#, diagnostics=[TaskAttempt 0 failed, info=[Error: Error while running task ( failure ) : attempt_#ID#:java.lang.RuntimeException: org.apache.hadoop.hive.ql.exec.mapjoin.MapJoinMemoryExhaustionError: assert_true_oom: assertation failed; Simulated OOM +#### A masked pattern was here #### +], TaskAttempt 1 failed, info=[Error: Error while running task ( failure ) : attempt_#ID#:java.lang.RuntimeException: org.apache.hadoop.hive.ql.exec.mapjoin.MapJoinMemoryExhaustionError: assert_true_oom: assertation failed; Simulated OOM +#### A masked pattern was here #### +]], Vertex did not succeed due to OWN_TASK_FAILURE, failedTasks:1 killedTasks:0, Vertex vertex_#ID# [Reducer 3] killed/failed due to:OWN_TASK_FAILURE] +DAG did not succeed due to VERTEX_FAILURE. failedVertices:1 killedVertices:0 +FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.tez.TezTask. Vertex failed, vertexName=Reducer 3, vertexId=vertex_#ID#, diagnostics=[Task failed, taskId=task_#ID#, diagnostics=[TaskAttempt 0 failed, info=[Error: Error while running task ( failure ) : attempt_#ID#:java.lang.RuntimeException: org.apache.hadoop.hive.ql.exec.mapjoin.MapJoinMemoryExhaustionError: assert_true_oom: assertation failed; Simulated OOM +#### A masked pattern was here #### +], TaskAttempt 1 failed, info=[Error: Error while running task ( failure ) : attempt_#ID#:java.lang.RuntimeException: org.apache.hadoop.hive.ql.exec.mapjoin.MapJoinMemoryExhaustionError: assert_true_oom: assertation failed; Simulated OOM +#### A masked pattern was here #### +]], Vertex did not succeed due to OWN_TASK_FAILURE, failedTasks:1 killedTasks:0, Vertex vertex_#ID# [Reducer 3] killed/failed due to:OWN_TASK_FAILURE]DAG did not succeed due to VERTEX_FAILURE. failedVertices:1 killedVertices:0 +PREHOOK: query: select assert_true_oom(2000 > sum(u*p)) from tx join px on (tx.a=px.a) where u<10 and p>2 +PREHOOK: type: QUERY +PREHOOK: Input: default@px +PREHOOK: Input: default@tx +#### A masked pattern was here #### +POSTHOOK: query: select assert_true_oom(2000 > sum(u*p)) from tx join px on (tx.a=px.a) where u<10 and p>2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@px +POSTHOOK: Input: default@tx +#### A masked pattern was here #### +NULL diff --git ql/src/test/results/clientpositive/llap/tez_input_counters.q.out ql/src/test/results/clientpositive/llap/tez_input_counters.q.out index 71d14f4049..ba517a21a3 100644 --- ql/src/test/results/clientpositive/llap/tez_input_counters.q.out +++ ql/src/test/results/clientpositive/llap/tez_input_counters.q.out @@ -1576,6 +1576,13 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1000 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 1000 + RECORDS_OUT_OPERATOR_SEL_1: 1000 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 309 @@ -1595,6 +1602,13 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 8 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_7: 1 + RECORDS_OUT_OPERATOR_GBY_5: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_4: 8 + RECORDS_OUT_OPERATOR_SEL_2: 8 + RECORDS_OUT_OPERATOR_TS_0: 8 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 2 @@ -1686,6 +1700,13 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 240 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 240 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_7: 1 + RECORDS_OUT_OPERATOR_GBY_5: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_4: 240 + RECORDS_OUT_OPERATOR_SEL_2: 240 + RECORDS_OUT_OPERATOR_TS_0: 240 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 74 @@ -1783,6 +1804,16 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 74 RECORDS_OUT_1_default.testpart1: 240 RECORDS_OUT_INTERMEDIATE_Map_1: 240 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_3: 240 + RECORDS_OUT_OPERATOR_FS_5: 74 + RECORDS_OUT_OPERATOR_GBY_3: 74 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_2: 240 + RECORDS_OUT_OPERATOR_SEL_1: 240 + RECORDS_OUT_OPERATOR_SEL_2: 240 + RECORDS_OUT_OPERATOR_SEL_4: 74 + RECORDS_OUT_OPERATOR_TS_0: 240 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 74 @@ -2291,6 +2322,22 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_INTERMEDIATE_Map_1: 1000 RECORDS_OUT_INTERMEDIATE_Map_4: 240 RECORDS_OUT_INTERMEDIATE_Reducer_2: 952 + RECORDS_OUT_INTERMEDIATE_Reducer_3: 0 + RECORDS_OUT_OPERATOR_EVENT_24: 309 + RECORDS_OUT_OPERATOR_FS_14: 1 + RECORDS_OUT_OPERATOR_GBY_12: 1 + RECORDS_OUT_OPERATOR_GBY_23: 309 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_MERGEJOIN_25: 952 + RECORDS_OUT_OPERATOR_RS_11: 952 + RECORDS_OUT_OPERATOR_RS_6: 1000 + RECORDS_OUT_OPERATOR_RS_7: 240 + RECORDS_OUT_OPERATOR_SEL_2: 1000 + RECORDS_OUT_OPERATOR_SEL_22: 1000 + RECORDS_OUT_OPERATOR_SEL_5: 240 + RECORDS_OUT_OPERATOR_SEL_9: 952 + RECORDS_OUT_OPERATOR_TS_0: 1000 + RECORDS_OUT_OPERATOR_TS_3: 240 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 GROUPED_INPUT_SPLITS_Map_4: 1 diff --git ql/src/test/results/clientpositive/show_functions.q.out ql/src/test/results/clientpositive/show_functions.q.out index 4e8ceaf6af..9a0930c3eb 100644 --- ql/src/test/results/clientpositive/show_functions.q.out +++ ql/src/test/results/clientpositive/show_functions.q.out @@ -31,6 +31,7 @@ array_contains ascii asin assert_true +assert_true_oom atan avg base64 diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/hooks/URIResolverHook.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/hooks/URIResolverHook.java index d3be5dd0a2..244ca8f2c1 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/hooks/URIResolverHook.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/hooks/URIResolverHook.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateDatabaseMessage.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateDatabaseMessage.java index 3d64c7341a..7cdeeda657 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateDatabaseMessage.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateDatabaseMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information