diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java index a3f56f75c4..d6b936d20a 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java @@ -204,7 +204,7 @@ public void testBasicBootstrapWithIncludeList() throws Throwable { createTables(originalMMAcidTables, CreateTableType.MM_ACID); // Replicate and verify if only 2 tables are replicated to target. - String replPolicy = primaryDbName + ".['t1', 't4', 't5']"; + String replPolicy = primaryDbName + ".'(t1)|(t4)|(t5)'"; String[] replicatedTables = new String[] {"t1", "t4", "t5"}; replicateAndVerify(replPolicy, null, null, null, replicatedTables); } @@ -215,7 +215,7 @@ public void testBasicBootstrapWithIncludeAndExcludeList() throws Throwable { createTables(originalTables, CreateTableType.NON_ACID); // Replicate and verify if only 3 tables are replicated to target. - String replPolicy = primaryDbName + ".['t1*', 't3'].['t100']"; + String replPolicy = primaryDbName + ".'(t1*)|(t3)'.'t100'"; String[] replicatedTables = new String[] {"t1", "t11", "t3"}; replicateAndVerify(replPolicy, null, null, null, replicatedTables); } @@ -234,7 +234,7 @@ public void testBasicIncrementalWithIncludeList() throws Throwable { createTables(originalMMAcidTables, CreateTableType.MM_ACID); // Replicate and verify if only 2 tables are replicated to target. - String replPolicy = primaryDbName + ".['t1', 't5']"; + String replPolicy = primaryDbName + ".'t1|t5'"; String[] replicatedTables = new String[] {"t1", "t5"}; replicateAndVerify(replPolicy, tupleBootstrap.lastReplicationId, null, null, replicatedTables); } @@ -249,7 +249,7 @@ public void testBasicIncrementalWithIncludeAndExcludeList() throws Throwable { createTables(originalTables, CreateTableType.NON_ACID); // Replicate and verify if only 3 tables are replicated to target. - String replPolicy = primaryDbName + ".['t1+', 't2'].['t11', 't3']"; + String replPolicy = primaryDbName + ".'(t1+)|(t2)'.'t11|t3'"; String[] replicatedTables = new String[] {"t1", "t111", "t2"}; replicateAndVerify(replPolicy, tupleBootstrap.lastReplicationId, null, null, replicatedTables); } @@ -263,12 +263,12 @@ public void testIncorrectTablePolicyInReplDump() throws Throwable { String[] replicatedTables = new String[] {}; boolean failed; String[] invalidReplPolicies = new String[] { - primaryDbName + ".t1.t2", // Two explicit table names not allowed. - primaryDbName + ".['t1'].t2", // Table name and include list not allowed. - primaryDbName + ".t1.['t2']", // Table name and exclude list not allowed. - primaryDbName + ".[t1].t2", // Table name and include list not allowed. - primaryDbName + ".['t1+'].", // Abrubtly ended dot. - primaryDbName + "..[]" // Multiple dots + primaryDbName + ".t1.t2", // Didn't enclose table pattern within single quotes. + primaryDbName + ".'t1'.t2", // Table name and include list not allowed. + primaryDbName + ".t1.'t2'", // Table name and exclude list not allowed. + primaryDbName + ".'t1+'.", // Abrubtly ended dot. + primaryDbName + ".['t1+'].['t11']", // With square brackets + primaryDbName + "..''" // Multiple dots }; for (String replPolicy : invalidReplPolicies) { failed = false; @@ -302,7 +302,7 @@ public void testIncorrectTablePolicyInReplDump() throws Throwable { // Replace with replication policy having different DB name. String oldReplPolicy = replPolicy; - replPolicy = primaryDbName + "_dupe.['t1+'].['t1']"; + replPolicy = primaryDbName + "_dupe.'t1+'.'t1'"; failed = false; try { replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, null, null, null, replicatedTables); @@ -313,8 +313,8 @@ public void testIncorrectTablePolicyInReplDump() throws Throwable { } Assert.assertTrue(failed); - // Invalid pattern where we didn't enclose table pattern within single or double quotes. - replPolicy = primaryDbName + ".[t1].[t2]"; + // Invalid pattern where where table list is empty. + replPolicy = primaryDbName + ".''.'t2'"; failed = false; try { replicateAndVerify(replPolicy, null, null, null, replicatedTables); @@ -334,8 +334,8 @@ public void testFullDbBootstrapReplicationWithDifferentReplPolicyFormats() throw // List of repl policy formats that leads to Full DB replication. String[] fullDbReplPolicies = new String[] { - primaryDbName + ".['.*?']", - primaryDbName + ".['.*?'].[]" + primaryDbName + ".'.*?'", + primaryDbName }; // Replicate and verify if all 3 tables are replicated to target. @@ -350,13 +350,13 @@ public void testCaseInSensitiveNatureOfReplPolicy() throws Throwable { createTables(originalTables, CreateTableType.NON_ACID); // Replicate and verify if 2 tables are replicated as per policy. - String replPolicy = primaryDbName.toUpperCase() + ".['.*a1+', 'cc3', 'B2'].['AA1+', 'b2']"; + String replPolicy = primaryDbName.toUpperCase() + ".'(.*a1+)|(cc3)|(B2)'.'(AA1+)|(b2)'"; String[] replicatedTables = new String[] {"a1", "cc3"}; String lastReplId = replicateAndVerify(replPolicy, null, null, null, replicatedTables); // Test case insensitive nature in REPLACE clause as well. String oldReplPolicy = replPolicy; - replPolicy = primaryDbName + ".['.*a1+', 'cc3', 'B2'].['AA1+']"; + replPolicy = primaryDbName + ".'(.*a1+)|(cc3)|(B2)'.'AA1+'"; replicatedTables = new String[] {"a1", "b2", "cc3"}; String[] bootstrappedTables = new String[] {"b2"}; replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, null, null, bootstrappedTables, replicatedTables); @@ -374,7 +374,7 @@ public void testBootstrapAcidTablesIncrementalPhaseWithIncludeAndExcludeList() t // Replicate and verify if only non-acid tables are replicated to target. List dumpWithoutAcidClause = Collections.singletonList( "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='false'"); - String replPolicy = primaryDbName + ".['a[0-9]+', 'b1'].['a4']"; + String replPolicy = primaryDbName + ".'(a[0-9]+)|(b1)'.'a4'"; String[] bootstrapReplicatedTables = new String[] {"a1"}; String lastReplId = replicateAndVerify(replPolicy, null, dumpWithoutAcidClause, null, bootstrapReplicatedTables); @@ -398,7 +398,7 @@ public void testBootstrapExternalTablesWithIncludeAndExcludeList() throws Throwa List dumpWithClause = Collections.singletonList( "'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'" ); - String replPolicy = primaryDbName + ".['a[0-9]+', 'b2'].['a1']"; + String replPolicy = primaryDbName + ".'(a[0-9]+)|(b2)'.'a1'"; String[] replicatedTables = new String[] {"a2", "b2"}; WarehouseInstance.Tuple tuple = primary.run("use " + primaryDbName) .dump(replPolicy, null, dumpWithClause); @@ -429,7 +429,7 @@ public void testBootstrapExternalTablesIncrementalPhaseWithIncludeAndExcludeList List dumpWithClause = Collections.singletonList( "'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='false'" ); - String replPolicy = primaryDbName + ".['a[0-9]+', 'b2'].['a1']"; + String replPolicy = primaryDbName + ".'(a[0-9]+)|(b2)'.'a1'"; String[] bootstrapReplicatedTables = new String[] {"b2"}; String lastReplId = replicateAndVerify(replPolicy, null, dumpWithClause, loadWithClause, bootstrapReplicatedTables); @@ -465,7 +465,7 @@ public void testBasicReplaceReplPolicy() throws Throwable { createTables(originalMMAcidTables, CreateTableType.MM_ACID); // Replicate and verify if only 2 tables are replicated to target. - String replPolicy = primaryDbName + ".['t1', 't4']"; + String replPolicy = primaryDbName + ".'(t1)|(t4)'"; String oldReplPolicy = null; String[] replicatedTables = new String[] {"t1", "t4"}; String lastReplId = replicateAndVerify(replPolicy, null, null, null, replicatedTables); @@ -473,7 +473,7 @@ public void testBasicReplaceReplPolicy() throws Throwable { // Exclude t4 and include t3, t6 createTables(new String[] {"t6"}, CreateTableType.MM_ACID); oldReplPolicy = replPolicy; - replPolicy = primaryDbName + ".['t1', 't3', 't6']"; + replPolicy = primaryDbName + ".'t1|t3|t6'"; replicatedTables = new String[] {"t1", "t3", "t6"}; String[] bootstrappedTables = new String[] {"t3", "t6"}; lastReplId = replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, @@ -489,7 +489,7 @@ public void testBasicReplaceReplPolicy() throws Throwable { // Convert to regex that excludes t3, t4 and t5. oldReplPolicy = replPolicy; - replPolicy = primaryDbName + ".['.*?'].['t[3-5]+']"; + replPolicy = primaryDbName + ".'.*?'.'t[3-5]+'"; replicatedTables = new String[] {"t1", "t2", "t6"}; bootstrappedTables = new String[] {}; replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, @@ -508,7 +508,7 @@ public void testReplacePolicyOnBootstrapAcidTablesIncrementalPhase() throws Thro // Replicate and verify if only non-acid tables are replicated to target. List dumpWithoutAcidClause = Collections.singletonList( "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='false'"); - String replPolicy = primaryDbName + ".['a[0-9]+', 'b[0-9]+'].['b1']"; + String replPolicy = primaryDbName + ".'(a[0-9]+)|(b[0-9]+)'.'b1'"; String[] bootstrapReplicatedTables = new String[] {"a1"}; String lastReplId = replicateAndVerify(replPolicy, null, dumpWithoutAcidClause, null, bootstrapReplicatedTables); @@ -516,7 +516,7 @@ public void testReplacePolicyOnBootstrapAcidTablesIncrementalPhase() throws Thro // Enable acid tables for replication. Also, replace, replication policy to exclude "b1" and "a3" // instead of "a1" alone. String oldReplPolicy = replPolicy; - replPolicy = primaryDbName + ".['[a-z]+[0-9]+'].['a3', 'b1']"; + replPolicy = primaryDbName + ".'[a-z]+[0-9]+'.'(a3)|(b1)'"; List dumpWithAcidBootstrapClause = Arrays.asList( "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='true'", "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES + "'='true'"); @@ -536,7 +536,7 @@ public void testReplacePolicyWhenAcidTablesDisabledForRepl() throws Throwable { // Replicate and verify if only non-acid tables are replicated to target. List dumpWithoutAcidClause = Collections.singletonList( "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='false'"); - String replPolicy = primaryDbName + ".['a[0-9]+', 'b[0-9]+'].['b1']"; + String replPolicy = primaryDbName + ".'(a[0-9]+)|(b[0-9]+)'.'b1'"; String[] bootstrapReplicatedTables = new String[] {"a1"}; String lastReplId = replicateAndVerify(replPolicy, null, dumpWithoutAcidClause, null, bootstrapReplicatedTables); @@ -545,7 +545,7 @@ public void testReplacePolicyWhenAcidTablesDisabledForRepl() throws Throwable { // "a2" but exclude "a1" and "b1". Still ACID tables shouldn't be bootstrapped. Only non-ACID // table "b1" should be bootstrapped. String oldReplPolicy = replPolicy; - replPolicy = primaryDbName + ".['a[0-9]+', 'b[0-9]+'].['a2']"; + replPolicy = primaryDbName + ".'(a[0-9]+)|(b[0-9]+)'.'a2'"; String[] incrementalReplicatedTables = new String[] {"a1", "b1"}; String[] bootstrappedTables = new String[] {"b1"}; lastReplId = replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, @@ -564,7 +564,7 @@ public void testReplacePolicyOnBootstrapExternalTablesIncrementalPhase() throws List dumpWithClause = Collections.singletonList( "'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='false'" ); - String replPolicy = primaryDbName + ".['a[0-9]+', 'b1'].['a1']"; + String replPolicy = primaryDbName + ".'(a[0-9]+)|(b1)'.'a1'"; String[] bootstrapReplicatedTables = new String[] {"b1"}; String lastReplId = replicateAndVerify(replPolicy, null, dumpWithClause, loadWithClause, bootstrapReplicatedTables); @@ -572,7 +572,7 @@ public void testReplacePolicyOnBootstrapExternalTablesIncrementalPhase() throws // Continue to disable external tables for replication. Also, replace, replication policy to exclude // "b1" and include "a1". String oldReplPolicy = replPolicy; - replPolicy = primaryDbName + ".['a[0-9]+', 'b[0-9]+'].['a2', 'b1']"; + replPolicy = primaryDbName + ".'(a[0-9]+)|(b[0-9]+)'.'(a2)|(b1)'"; String[] incrementalReplicatedTables = new String[] {"a1"}; String[] bootstrappedTables = new String[] {"a1"}; lastReplId = replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, @@ -581,7 +581,7 @@ public void testReplacePolicyOnBootstrapExternalTablesIncrementalPhase() throws // Enable external tables replication and bootstrap in incremental phase. Also, replace, // replication policy to exclude tables with prefix "b". oldReplPolicy = replPolicy; - replPolicy = primaryDbName + ".['[a-z]+[0-9]+'].['b[0-9]+']"; + replPolicy = primaryDbName + ".'[a-z]+[0-9]+'.'b[0-9]+'"; incrementalReplicatedTables = new String[] {"a1", "a2", "c2"}; bootstrappedTables = new String[] {"a2", "c2"}; dumpWithClause = Arrays.asList("'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'", @@ -608,7 +608,7 @@ public void testReplacePolicyOnBootstrapExternalTablesIncrementalPhase() throws @Test public void testRenameTableScenariosBasic() throws Throwable { - String replPolicy = primaryDbName + ".['in[0-9]+'].['out[0-9]+']"; + String replPolicy = primaryDbName + ".'in[0-9]+'.'out[0-9]+'"; String lastReplId = replicateAndVerify(replPolicy, null, null, null, null, new String[] {}, new String[] {}); @@ -662,7 +662,7 @@ public void testRenameTableScenariosBasic() throws Throwable { @Test public void testRenameTableScenariosWithDmlOperations() throws Throwable { - String replPolicy = primaryDbName + ".['in[0-9]+'].['out[0-9]+']"; + String replPolicy = primaryDbName + ".'in[0-9]+'.'out[0-9]+'"; String lastReplId = replicateAndVerify(replPolicy, null, null, null, null, new String[] {}, new String[] {}); @@ -694,7 +694,7 @@ public void testRenameTableScenariosWithDmlOperations() throws Throwable { @Test public void testRenameTableScenariosAcidTable() throws Throwable { - String replPolicy = primaryDbName + ".['in[0-9]+'].['out[0-9]+']"; + String replPolicy = primaryDbName + ".'in[0-9]+'.'out[0-9]+'"; List dumpWithClause = Arrays.asList( "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES.varname + "'='false'", "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='false'" @@ -731,7 +731,7 @@ public void testRenameTableScenariosAcidTable() throws Throwable { @Test public void testRenameTableScenariosExternalTable() throws Throwable { - String replPolicy = primaryDbName + ".['in[0-9]+'].['out[0-9]+']"; + String replPolicy = primaryDbName + ".'in[0-9]+'.'out[0-9]+'"; List loadWithClause = ReplicationTestUtils.externalTableBasePathWithClause(REPLICA_EXTERNAL_BASE, replica); List dumpWithClause = Arrays.asList( "'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='false'", @@ -794,7 +794,7 @@ public void testRenameTableScenariosWithReplaceExternalTable() throws Throwable "'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'", "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname + "'='true'" ); - String replPolicy = primaryDbName + ".['in[0-9]+', 'out4', 'out5', 'out1500']"; + String replPolicy = primaryDbName + ".'(in[0-9]+)|(out4)|(out5)|(out1500)'"; String lastReplId = replicateAndVerify(replPolicy, null, null, dumpWithClause, loadWithClause, new String[] {}, new String[] {}); @@ -813,7 +813,7 @@ public void testRenameTableScenariosWithReplaceExternalTable() throws Throwable .run("alter table out11 rename to out12") // Old name not matching old and new name not matching both .run("alter table out12 rename to in12"); // Old name not matching old and new name matching both - String newPolicy = primaryDbName + ".['in[0-9]+', 'out1500'].['in2']"; + String newPolicy = primaryDbName + ".'(in[0-9]+)|(out1500)|(in2)'"; dumpWithClause = Arrays.asList( "'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'", "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname + "'='false'" @@ -828,7 +828,7 @@ public void testRenameTableScenariosWithReplaceExternalTable() throws Throwable @Test public void testRenameTableScenariosWithReplacePolicyDMLOperattion() throws Throwable { - String replPolicy = primaryDbName + ".['in[0-9]+', 'out5000', 'out5001'].['in100', 'in200', 'in305']"; + String replPolicy = primaryDbName + ".'(in[0-9]+)|(out5000)|(out5001)'.'(in100)|(in200)|(in305)'"; String lastReplId = replicateAndVerify(replPolicy, null, null, null, null, new String[] {}, new String[] {}); @@ -845,7 +845,7 @@ public void testRenameTableScenariosWithReplacePolicyDMLOperattion() throws Thro null, bootstrapTables, replicatedTables); // Rename the tables to satisfy the condition also replace the policy. - String newPolicy = primaryDbName + ".['in[0-9]+', 'out3000'].['in2']"; + String newPolicy = primaryDbName + ".'(in[0-9]+)|(out3000)'.'in2'"; primary.run("use " + primaryDbName) .run("alter table in200 rename to in2000") // Old name not matching old, old and new matching new policy. .run("alter table in400 rename to out400") // Old name matching new and old policy, new matching none. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index a4b233ad83..3a2a2d0527 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -939,26 +939,10 @@ replTableLevelPolicy @init { pushMsg("Replication table level policy definition", state); } @after { popMsg(state); } : - ((replTablesIncludeList=replTablesList) (DOT replTablesExcludeList=replTablesList)?) + ((replTablesIncludeList=StringLiteral) (DOT replTablesExcludeList=StringLiteral)?) -> ^(TOK_REPL_TABLES $replTablesIncludeList $replTablesExcludeList?) ; -replTablesList -@init { pushMsg("replication table name or comma separated table names pattern list", state); } -@after { popMsg(state); } - : - (LSQUARE (tablePattern (COMMA tablePattern)*)? RSQUARE) -> ^(TOK_REPL_TABLES_LIST tablePattern*) - ; - -tablePattern -@init { pushMsg("Table name pattern", state); } -@after { popMsg(state); } - : - (pattern=StringLiteral) -> $pattern - | - (identifier) -> TOK_NULL - ; - replStatusStatement @init { pushMsg("replication status statement", state); } @after { popMsg(state); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java index df41a2e978..695443cfd3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java @@ -140,25 +140,16 @@ private void setReplDumpTablesList(Tree replTablesNode, ReplScope replScope) thr // and exclude tables lists. String replScopeType = (replScope == this.replScope) ? "Current" : "Old"; for (int listIdx = 0; listIdx < childCount; listIdx++) { - Tree tablesListNode = replTablesNode.getChild(listIdx); - assert(tablesListNode.getType() == TOK_REPL_TABLES_LIST); - - List tablesList = new ArrayList<>(); - for (int child = 0; child < tablesListNode.getChildCount(); child++) { - Tree tablePatternNode = tablesListNode.getChild(child); - if (tablePatternNode.getType() == TOK_NULL) { - LOG.error(ErrorMsg.REPL_INVALID_DB_OR_TABLE_PATTERN.getMsg()); - throw new SemanticException(ErrorMsg.REPL_INVALID_DB_OR_TABLE_PATTERN.getMsg()); - } - tablesList.add(unescapeSQLString(tablePatternNode.getText())); + String tableList = unescapeSQLString(replTablesNode.getChild(listIdx).getText()); + if (tableList == null || tableList.isEmpty()) { + throw new SemanticException(ErrorMsg.REPL_INVALID_DB_OR_TABLE_PATTERN); } - if (listIdx == 0) { - LOG.info("{} ReplScope: Set Included Tables List: {}", replScopeType, tablesList); - replScope.setIncludedTablePatterns(tablesList); + LOG.info("{} ReplScope: Set Included Tables List: {}", replScopeType, tableList); + replScope.setIncludedTablePatterns(tableList); } else { - LOG.info("{} ReplScope: Set Excluded Tables List: {}", replScopeType, tablesList); - replScope.setExcludedTablePatterns(tablesList); + LOG.info("{} ReplScope: Set Excluded Tables List: {}", replScopeType, tableList); + replScope.setExcludedTablePatterns(tableList); } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java index 4c137eb365..f61e99c1e7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java @@ -85,48 +85,27 @@ private void readReplScope(String line) throws IOException { } String[] lineContents = line.split("\t"); - if (lineContents.length < 1) { - return; - } - replScope = new ReplScope(); - - LOG.info("Read ReplScope: Set Db Name: {}.", lineContents[0]); - replScope.setDbName(lineContents[0]); - - // Read/set include and exclude tables list. - int idx = readReplScopeTablesList(lineContents, 1, true); - readReplScopeTablesList(lineContents, idx, false); - } - - private int readReplScopeTablesList(String[] lineContents, int startIdx, boolean includeList) - throws IOException { - // If the list doesn't exist, then return. - if (startIdx >= lineContents.length) { - return startIdx; - } - - // Each tables list should start with "{" and ends with "}" - if (!"{".equals(lineContents[startIdx])) { - throw new IOException("Invalid repl tables list data in dump metadata file. Missing \"{\"."); - } - - ListtableNames = new ArrayList<>(); - for (int i = (startIdx + 1); i < lineContents.length; i++) { - String value = lineContents[i]; - if ("}".equals(value)) { - if (includeList) { - LOG.info("Read ReplScope: Set Include Table Names: {}.", tableNames); - replScope.setIncludedTablePatterns(tableNames); - } else { - LOG.info("Read ReplScope: Set Exclude Table Names: {}.", tableNames); - replScope.setExcludedTablePatterns(tableNames); + for (int idx = 0; idx < lineContents.length; idx++) { + String value = lineContents[idx]; + switch (idx) { + case 0: { + LOG.info("Read ReplScope: Set Db Name: {}.", value); + replScope.setDbName(value); + break; + } + case 1: { + LOG.info("Read ReplScope: Include table name list: {}.", value); + replScope.setIncludedTablePatterns(value); + break; + } + case 2: { + LOG.info("Read ReplScope: Exclude table name list: {}.", value); + replScope.setExcludedTablePatterns(value); + break; } - return (i + 1); } - tableNames.add(value); } - throw new IOException("Invalid repl tables list data in dump metadata file. Missing \"}\"."); } private void loadDumpFromFile() throws SemanticException { @@ -205,17 +184,13 @@ private void initializeIfNot() throws SemanticException { List values = new ArrayList<>(); values.add(replScope.getDbName()); - List includedTableNames = replScope.getIncludedTableNames(); - List excludedTableNames = replScope.getExcludedTableNames(); + String includedTableNames = replScope.getIncludedTableNames(); + String excludedTableNames = replScope.getExcludedTableNames(); if (includedTableNames != null) { - values.add("{"); - values.addAll(includedTableNames); - values.add("}"); + values.add(includedTableNames); } if (excludedTableNames != null) { - values.add("{"); - values.addAll(excludedTableNames); - values.add("}"); + values.add(excludedTableNames); } LOG.info("Preparing ReplScope {} to dump.", values); return values; diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/repl/ReplScope.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/repl/ReplScope.java index 14b58a3f51..764113a23c 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/repl/ReplScope.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/repl/ReplScope.java @@ -18,8 +18,6 @@ package org.apache.hadoop.hive.common.repl; import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; import java.util.regex.Pattern; /** @@ -31,10 +29,10 @@ private Pattern dbNamePattern; // Include and exclude table names/patterns exist only for REPL DUMP. - private List includedTableNames; - private List excludedTableNames; - private List includedTableNamePatterns; - private List excludedTableNamePatterns; + private String includedTableNames; + private String excludedTableNames; + private Pattern includedTableNamePatterns; + private Pattern excludedTableNamePatterns; public ReplScope() { } @@ -53,21 +51,21 @@ public String getDbName() { return dbName; } - public void setIncludedTablePatterns(List includedTableNames) { + public void setIncludedTablePatterns(String includedTableNames) { this.includedTableNames = includedTableNames; this.includedTableNamePatterns = compilePatterns(includedTableNames); } - public List getIncludedTableNames() { + public String getIncludedTableNames() { return includedTableNames; } - public void setExcludedTablePatterns(List excludedTableNames) { + public void setExcludedTablePatterns(String excludedTableNames) { this.excludedTableNames = excludedTableNames; this.excludedTableNamePatterns = compilePatterns(excludedTableNames); } - public List getExcludedTableNames() { + public String getExcludedTableNames() { return excludedTableNames; } @@ -93,26 +91,13 @@ public boolean tableIncludedInReplScope(final String tableName) { return (inTableIncludedList(tableName) && !inTableExcludedList(tableName)); } - private List compilePatterns(List patterns) { - if (patterns == null || patterns.isEmpty()) { + private Pattern compilePatterns(String pattern) { + if (pattern == null) { return null; } - List compiledPatterns = new ArrayList<>(); - for (String pattern : patterns) { - // Convert the pattern to lower case because events/HMS will have table names in lower case. - compiledPatterns.add(Pattern.compile(pattern, Pattern.CASE_INSENSITIVE)); - } - return compiledPatterns; - } - private boolean tableMatchAnyPattern(final String tableName, List tableNamePatterns) { - assert(tableNamePatterns != null); - for (Pattern tableNamePattern : tableNamePatterns) { - if (tableNamePattern.matcher(tableName).matches()) { - return true; - } - } - return false; + // Convert the pattern to lower case because events/HMS will have table names in lower case. + return Pattern.compile(pattern, Pattern.CASE_INSENSITIVE); } private boolean inTableIncludedList(final String tableName) { @@ -121,7 +106,7 @@ private boolean inTableIncludedList(final String tableName) { // So, all tables must be included. return true; } - return tableMatchAnyPattern(tableName, includedTableNamePatterns); + return includedTableNamePatterns.matcher(tableName).matches(); } private boolean inTableExcludedList(final String tableName) { @@ -129,6 +114,6 @@ private boolean inTableExcludedList(final String tableName) { // If excluded tables list is empty means, all tables in included list must be accepted. return false; } - return tableMatchAnyPattern(tableName, excludedTableNamePatterns); + return excludedTableNamePatterns.matcher(tableName).matches(); } }