diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java index 1fce791cbe..b3aa52b223 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java @@ -54,7 +54,6 @@ import org.apache.hadoop.hive.ql.exec.util.DAGTraversal; import org.apache.hadoop.hive.ql.exec.util.Retryable; import org.apache.hadoop.hive.ql.io.AcidUtils; -import org.apache.hadoop.hive.ql.io.orc.ExternalCache; import org.apache.hadoop.hive.ql.lockmgr.DbLockManager; import org.apache.hadoop.hive.ql.lockmgr.HiveLockManager; import org.apache.hadoop.hive.ql.lockmgr.LockException; @@ -164,6 +163,7 @@ public int execute() { Path previousValidHiveDumpPath = getPreviousValidDumpMetadataPath(dumpRoot); boolean isBootstrap = (previousValidHiveDumpPath == null); work.setBootstrap(isBootstrap); + work.setOldReplScope(new DumpMetaData(previousValidHiveDumpPath, conf).getReplScope()); //If no previous dump is present or previous dump is already loaded, proceed with the dump operation. if (shouldDump(previousValidHiveDumpPath)) { Path currentDumpPath = getCurrentDumpPath(dumpRoot, isBootstrap); @@ -411,12 +411,16 @@ private boolean shouldDump(Path previousDumpPath) throws IOException { * @return true if need to examine tables for dump and false if not. */ private boolean shouldExamineTablesToDump() { - return (work.oldReplScope != null) + return (previousReplScopeModified()) || !tablesForBootstrap.isEmpty() || conf.getBoolVar(HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES) || conf.getBoolVar(HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES); } + private boolean previousReplScopeModified() { + return !(work.oldReplScope.equals(work.replScope)); + } + /** * Decide whether to dump external tables data. If external tables are enabled for replication, * then need to dump it's data in all the incremental dumps. @@ -764,7 +768,7 @@ private boolean needBootstrapAcidTablesDuringIncrementalDump() { // may not satisfying the old policy but satisfying the new policy. For filter, it may happen that the table // is renamed and started satisfying the policy. return ((!work.replScope.includeAllTables()) - || (work.oldReplScope != null) + || (previousReplScopeModified()) || conf.getBoolVar(HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES)); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpWork.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpWork.java index aecfe757a4..47051e41ee 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpWork.java @@ -24,7 +24,6 @@ import org.apache.hadoop.hive.ql.exec.ReplCopyTask; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; -import org.apache.hadoop.hive.ql.exec.repl.util.FileList; import org.apache.hadoop.hive.ql.exec.repl.util.TaskTracker; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.parse.EximUtil; @@ -47,7 +46,7 @@ private static final long serialVersionUID = 1L; private static final Logger LOG = LoggerFactory.getLogger(ReplDumpWork.class); final ReplScope replScope; - final ReplScope oldReplScope; + ReplScope oldReplScope; final String dbNameOrPattern, astRepresentationForErrorMsg, resultTempPath; Long eventTo; Long eventFrom; @@ -88,16 +87,19 @@ public static void testDeletePreviousDumpMetaPath(boolean failDeleteDumpMeta) { testDeletePreviousDumpMetaPath = failDeleteDumpMeta; } - public ReplDumpWork(ReplScope replScope, ReplScope oldReplScope, + public ReplDumpWork(ReplScope replScope, String astRepresentationForErrorMsg, String resultTempPath) { this.replScope = replScope; - this.oldReplScope = oldReplScope; this.dbNameOrPattern = replScope.getDbName(); this.astRepresentationForErrorMsg = astRepresentationForErrorMsg; this.resultTempPath = resultTempPath; } + void setOldReplScope(ReplScope replScope) { + oldReplScope = replScope; + } + int maxEventLimit() throws Exception { if (eventTo < eventFrom) { throw new Exception("Invalid event ID input received in TO clause"); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java index 4c10499e5e..313474c7f6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java @@ -65,7 +65,6 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer { // Replication Scope private ReplScope replScope = new ReplScope(); - private ReplScope oldReplScope = null; // Source DB Name for REPL LOAD private String sourceDbNameOrPattern; @@ -143,34 +142,6 @@ private void setReplDumpTablesList(Tree replTablesNode, ReplScope replScope) thr } } - private void setOldReplPolicy(Tree oldReplPolicyTree) throws HiveException { - oldReplScope = new ReplScope(); - int childCount = oldReplPolicyTree.getChildCount(); - - // First child is DB name and optional second child is tables list. - assert(childCount <= 2); - - // First child is always the DB name. So set it. - oldReplScope.setDbName(oldReplPolicyTree.getChild(0).getText()); - LOG.info("Old ReplScope: Set DB Name: {}", oldReplScope.getDbName()); - if (!oldReplScope.getDbName().equalsIgnoreCase(replScope.getDbName())) { - LOG.error("DB name {} cannot be replaced to {} in the replication policy.", - oldReplScope.getDbName(), replScope.getDbName()); - throw new SemanticException("DB name cannot be replaced in the replication policy."); - } - - // If the old policy is just , then tables list won't be there. - if (childCount <= 1) { - return; - } - - // Traverse the children which can be either just include tables list or both include - // and exclude tables lists. - Tree oldPolicyTablesListNode = oldReplPolicyTree.getChild(1); - assert(oldPolicyTablesListNode.getType() == TOK_REPL_TABLES); - setReplDumpTablesList(oldPolicyTablesListNode, oldReplScope); - } - private void initReplDump(ASTNode ast) throws HiveException { int numChildren = ast.getChildCount(); boolean isMetaDataOnly = false; @@ -196,9 +167,6 @@ private void initReplDump(ASTNode ast) throws HiveException { case TOK_REPL_TABLES: setReplDumpTablesList(currNode, replScope); break; - case TOK_REPLACE: - setOldReplPolicy(currNode); - break; default: throw new SemanticException("Unrecognized token " + currNode.getType() + " in REPL DUMP statement."); } @@ -237,7 +205,6 @@ private void analyzeReplDump(ASTNode ast) throws SemanticException { Task replDumpWorkTask = TaskFactory .get(new ReplDumpWork( replScope, - oldReplScope, ASTErrorUtils.getMsg(ErrorMsg.INVALID_PATH.getMsg(), ast), ctx.getResFile().toUri().toString() ), conf); diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/repl/ReplScope.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/repl/ReplScope.java index 8b26f1ce40..e4337c4286 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/repl/ReplScope.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/repl/ReplScope.java @@ -116,4 +116,20 @@ private boolean inTableExcludedList(final String tableName) { } return excludedTableNamePattern.matcher(tableName).matches(); } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (! (o instanceof ReplScope)) { + return false; + } + + ReplScope replScope = (ReplScope)o; + return (replScope.getExcludedTableNames().equals(this.excludedTableNames) + && replScope.getIncludedTableNames().equals(this.includedTableNames) + && replScope.dbName.equals(this.dbName) + && replScope.dbNamePattern.equals(this.dbNamePattern)); + } }