diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java index f973f8d..456e80d 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java @@ -50,6 +50,10 @@ import org.junit.Test; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import static org.mockito.Mockito.when; /** * Test HiveAuthorizer api invocation @@ -390,6 +394,40 @@ public void testDescDatabase() throws HiveAuthzPluginException, assertEquals(dbName.toLowerCase(), dbObj.getDbname().toLowerCase()); } + private void resetAuthorizer() throws HiveAuthzPluginException, HiveAccessControlException { + reset(mockedAuthorizer); + // HiveAuthorizer.filterListCmdObjects should not filter any object + when(mockedAuthorizer.filterListCmdObjects(any(List.class), + any(HiveAuthzContext.class))).thenAnswer(new Answer>() { + @Override + public List answer(InvocationOnMock invocation) throws Throwable { + Object[] args = invocation.getArguments(); + return (List) args[0]; + } + }); + } + + @Test + public void testReplDump() throws HiveAuthzPluginException, HiveAccessControlException, + CommandNeedRetryException { + + resetAuthorizer(); + int status = driver.compile("repl dump " + dbName); + assertEquals(0, status); + List inputs = getHivePrivilegeObjectInputs().getLeft(); + HivePrivilegeObject dbObj = inputs.get(0); + assertEquals("input type", HivePrivilegeObjectType.DATABASE, dbObj.getType()); + assertEquals("db name", dbName.toLowerCase(), dbObj.getDbname()); + + resetAuthorizer(); + status = driver.compile("repl dump " + dbName + "." + inDbTableName); + assertEquals(0, status); + inputs = getHivePrivilegeObjectInputs().getLeft(); + dbObj = inputs.get(0); + assertEquals("input type", HivePrivilegeObjectType.TABLE_OR_VIEW, dbObj.getType()); + assertEquals("db name", dbName.toLowerCase(), dbObj.getDbname()); + assertEquals("table name", inDbTableName.toLowerCase(), dbObj.getObjectName()); + } private void checkSingleTableInput(List inputs) { assertEquals("number of inputs", 1, inputs.size()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java index f9bdff8..f3ba3e4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java @@ -17,7 +17,6 @@ Licensed to the Apache Software Foundation (ASF) under one */ package org.apache.hadoop.hive.ql.exec.repl; -import com.google.common.collect.Collections2; import com.google.common.primitives.Ints; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -40,7 +39,6 @@ Licensed to the Apache Software Foundation (ASF) under one import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec; import org.apache.hadoop.hive.ql.parse.EximUtil; import org.apache.hadoop.hive.ql.parse.ReplicationSpec; -import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.parse.repl.DumpType; import org.apache.hadoop.hive.ql.parse.repl.dump.HiveWrapper; @@ -65,9 +63,6 @@ Licensed to the Apache Software Foundation (ASF) under one private static final String FUNCTIONS_ROOT_DIR_NAME = "_functions"; private static final String FUNCTION_METADATA_DIR_NAME = "_metadata"; - private final static String TMP_TABLE_PREFIX = - SemanticAnalyzer.VALUES_TMP_TABLE_NAME_PREFIX.toLowerCase(); - private Logger LOG = LoggerFactory.getLogger(ReplDumpTask.class); private Logger REPL_STATE_LOG = LoggerFactory.getLogger("ReplState"); @@ -176,7 +171,7 @@ private ReplicationSpec getNewEventOnlyReplicationSpec(Long eventId) throws Sema private Long bootStrapDump(Path dumpRoot, DumpMetaData dmd, Path cmRoot) throws Exception { // bootstrap case Long bootDumpBeginReplId = getHive().getMSC().getCurrentNotificationEventId().getEventId(); - for (String dbName : matchesDb()) { + for (String dbName : Utils.matchesDb(getHive(), work.dbNameOrPattern)) { REPL_STATE_LOG .info("Repl Dump: Started analyzing Repl Dump for DB: {}, Dump Type: BOOTSTRAP", dbName); @@ -184,7 +179,7 @@ private Long bootStrapDump(Path dumpRoot, DumpMetaData dmd, Path cmRoot) throws Path dbRoot = dumpDbMetadata(dbName, dumpRoot); dumpFunctionMetadata(dbName, dumpRoot); - for (String tblName : matchesTbl(dbName, work.tableNameOrPattern)) { + for (String tblName : Utils.matchesTbl(getHive(), dbName, work.tableNameOrPattern)) { LOG.debug( "analyzeReplDump dumping table: " + tblName + " to db root " + dbRoot.toUri()); dumpTable(dbName, tblName, dbRoot); @@ -223,28 +218,6 @@ private Long bootStrapDump(Path dumpRoot, DumpMetaData dmd, Path cmRoot) throws return bootDumpEndReplId; } - private Iterable matchesDb() throws HiveException { - if (work.dbNameOrPattern == null) { - return getHive().getAllDatabases(); - } else { - return getHive().getDatabasesByPattern(work.dbNameOrPattern); - } - } - - private Iterable matchesTbl(String dbName, String tblPattern) - throws HiveException { - Hive db = Hive.get(); - if (tblPattern == null) { - return Collections2.filter(db.getAllTables(dbName), - tableName -> { - assert tableName != null; - return !tableName.toLowerCase().startsWith(TMP_TABLE_PREFIX); - }); - } else { - return db.getTablesByPattern(dbName, tblPattern); - } - } - private Path dumpDbMetadata(String dbName, Path dumpRoot) throws Exception { Path dbRoot = new Path(dumpRoot, dbName); // TODO : instantiating FS objects are generally costly. Refactor diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java index 48d9c94..b41facd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java @@ -23,14 +23,17 @@ Licensed to the Apache Software Foundation (ASF) under one import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.exec.repl.ReplDumpTask; import org.apache.hadoop.hive.ql.exec.repl.ReplDumpWork; import org.apache.hadoop.hive.ql.hooks.ReadEntity; +import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; @@ -174,6 +177,17 @@ private void analyzeReplDump(ASTNode ast) throws SemanticException { ctx.getResFile().toUri().toString() ), conf); rootTasks.add(replDumpWorkTask); + if (dbNameOrPattern != null) { + for (String dbName : Utils.matchesDb(db, dbNameOrPattern)) { + if (tblNameOrPattern != null) { + for (String tblName : Utils.matchesTbl(db, dbName, tblNameOrPattern)) { + inputs.add(new ReadEntity(db.getTable(dbName, tblName))); + } + } else { + inputs.add(new ReadEntity(db.getDatabase(dbName))); + } + } + } setFetchTask(createFetchTask(dumpSchema)); } catch (Exception e) { // TODO : simple wrap & rethrow for now, clean up with error codes diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java index 846b6f5..f40c703 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java @@ -21,9 +21,14 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.io.IOUtils; +import com.google.common.collect.Collections2; + import java.io.DataOutputStream; import java.io.IOException; import java.util.List; @@ -47,4 +52,26 @@ public static void writeOutput(List values, Path outputFile, HiveConf hi IOUtils.closeStream(outStream); } } + + public static Iterable matchesDb(Hive db, String dbPattern) throws HiveException { + if (dbPattern == null) { + return db.getAllDatabases(); + } else { + return db.getDatabasesByPattern(dbPattern); + } + } + + public static Iterable matchesTbl(Hive db, String dbName, String tblPattern) + throws HiveException { + if (tblPattern == null) { + return Collections2.filter(db.getAllTables(dbName), + tableName -> { + assert tableName != null; + return !tableName.toLowerCase().startsWith( + SemanticAnalyzer.VALUES_TMP_TABLE_NAME_PREFIX.toLowerCase()); + }); + } else { + return db.getTablesByPattern(dbName, tblPattern); + } + } } diff --git a/ql/src/test/results/clientnegative/repl_dump_requires_admin.q.out b/ql/src/test/results/clientnegative/repl_dump_requires_admin.q.out index 29e72f4..ac5103e 100644 --- a/ql/src/test/results/clientnegative/repl_dump_requires_admin.q.out +++ b/ql/src/test/results/clientnegative/repl_dump_requires_admin.q.out @@ -55,8 +55,10 @@ POSTHOOK: type: SHOWTABLES POSTHOOK: Input: database:default PREHOOK: query: repl dump test_repldump_adminpriv PREHOOK: type: REPLDUMP +PREHOOK: Input: database:test_repldump_adminpriv POSTHOOK: query: repl dump test_repldump_adminpriv POSTHOOK: type: REPLDUMP +POSTHOOK: Input: database:test_repldump_adminpriv #### A masked pattern was here #### PREHOOK: query: show tables test_repldump_adminpriv PREHOOK: type: SHOWTABLES diff --git a/ql/src/test/results/clientnegative/repl_load_requires_admin.q.out b/ql/src/test/results/clientnegative/repl_load_requires_admin.q.out index df0b5dc..16cd9b7 100644 --- a/ql/src/test/results/clientnegative/repl_load_requires_admin.q.out +++ b/ql/src/test/results/clientnegative/repl_load_requires_admin.q.out @@ -63,8 +63,10 @@ POSTHOOK: type: SHOWTABLES POSTHOOK: Input: database:default PREHOOK: query: repl dump test_replload_adminpriv_src PREHOOK: type: REPLDUMP +PREHOOK: Input: database:test_replload_adminpriv_src POSTHOOK: query: repl dump test_replload_adminpriv_src POSTHOOK: type: REPLDUMP +POSTHOOK: Input: database:test_replload_adminpriv_src #### A masked pattern was here #### PREHOOK: type: REPLLOAD #### A masked pattern was here ####