diff --git ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java index 650792b..154fec4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java +++ ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java @@ -283,6 +283,7 @@ public void clear() { analyzeRewrite = false; ctas = false; outerQueryLimit = -1; + isMaterializedView = false; hasJoin = false; hasGroupBy = false; diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java index 3e4584d..e1340c7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java @@ -79,7 +79,7 @@ public void initialize(HiveConf hiveConf) { if (postExecHooks.contains("org.apache.hadoop.hive.ql.hooks.PostExecutePrinter") || postExecHooks.contains("org.apache.hadoop.hive.ql.hooks.LineageLogger") || postExecHooks.contains("org.apache.atlas.hive.hook.HiveHook")) { - transformations.add(new Generator()); + transformations.add(new Generator(postExecHooks)); } // Try to transform OR predicates in Filter into simpler IN clauses first diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java index 747aae0..e6c0771 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; import org.apache.hadoop.hive.ql.exec.CommonJoinOperator; import org.apache.hadoop.hive.ql.exec.FilterOperator; @@ -59,12 +60,32 @@ private static final Logger LOG = LoggerFactory.getLogger(Generator.class); + private final Set hooks; + private static final String ATLAS_HOOK_CLASSNAME = "org.apache.atlas.hive.hook.HiveHook"; + + public Generator(Set hooks) { + this.hooks = hooks; + } + /* (non-Javadoc) * @see org.apache.hadoop.hive.ql.optimizer.Transform#transform(org.apache.hadoop.hive.ql.parse.ParseContext) */ @Override public ParseContext transform(ParseContext pctx) throws SemanticException { + if (hooks != null && hooks.contains(ATLAS_HOOK_CLASSNAME)) { + // Atlas would be interested in lineage information for insert,load,create etc. + if (!pctx.getQueryProperties().isCTAS() + && !pctx.getQueryProperties().isMaterializedView() + && pctx.getQueryProperties().isQuery() + && pctx.getCreateTable() == null + && pctx.getCreateViewDesc() == null + && (pctx.getLoadTableWork() == null || pctx.getLoadTableWork().isEmpty())) { + LOG.debug("Not evaluating lineage"); + return pctx; + } + } + Index index = SessionState.get() != null ? SessionState.get().getLineageState().getIndex() : new Index(); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index d7f7d48..d06a0b5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -11407,7 +11407,7 @@ void analyzeInternal(ASTNode ast, PlannerContext plannerCtx) throws SemanticExce || postExecHooks.contains("org.apache.atlas.hive.hook.HiveHook")) { ArrayList transformations = new ArrayList(); transformations.add(new HiveOpConverterPostProc()); - transformations.add(new Generator()); + transformations.add(new Generator(postExecHooks)); for (Transform t : transformations) { pCtx = t.transform(pCtx); }