diff --git a/upgrade-acid/pre-upgrade/src/main/java/org/apache/hadoop/hive/upgrade/acid/PreUpgradeTool.java b/upgrade-acid/pre-upgrade/src/main/java/org/apache/hadoop/hive/upgrade/acid/PreUpgradeTool.java index 04782a62ad..5731139ab0 100644 --- a/upgrade-acid/pre-upgrade/src/main/java/org/apache/hadoop/hive/upgrade/acid/PreUpgradeTool.java +++ b/upgrade-acid/pre-upgrade/src/main/java/org/apache/hadoop/hive/upgrade/acid/PreUpgradeTool.java @@ -25,6 +25,7 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -54,6 +55,7 @@ import org.apache.hadoop.hive.ql.io.orc.Reader; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.shims.HadoopShims; +import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hive.common.util.HiveVersionInfo; import org.apache.thrift.TException; @@ -223,6 +225,7 @@ private void prepareAcidUpgradeInternal(String scriptLocation, boolean execute) boolean isAcidEnabled = isAcidEnabled(conf); IMetaStoreClient hms = getHMS(conf); LOG.debug("Looking for databases"); + // if we cannot databases we cannot proceed with pre-upgrade so we don't handle exception here List databases = hms.getAllDatabases();//TException LOG.debug("Found " + databases.size() + " databases to process"); List compactions = new ArrayList<>(); @@ -234,26 +237,46 @@ private void prepareAcidUpgradeInternal(String scriptLocation, boolean execute) } for(String dbName : databases) { - List tables = hms.getAllTables(dbName); - LOG.debug("found " + tables.size() + " tables in " + dbName); - for(String tableName : tables) { - Table t = hms.getTable(dbName, tableName); - LOG.debug("processing table " + Warehouse.getQualifiedName(t)); - if(isAcidEnabled) { - //if acid is off, there can't be any acid tables - nothing to compact - if(txns == null) { + try { + List tables = hms.getAllTables(dbName); + LOG.debug("found " + tables.size() + " tables in " + dbName); + for (String tableName : tables) { + try { + Table t = hms.getTable(dbName, tableName); + LOG.debug("processing table " + Warehouse.getQualifiedName(t)); + if (isAcidEnabled) { + //if acid is off, there can't be any acid tables - nothing to compact + if (txns == null) { /* This API changed from 2.x to 3.0. so this won't even compile with 3.0 but it doesn't need to since we only run this preUpgrade */ - TxnStore txnHandler = TxnUtils.getTxnStore(conf); - txns = TxnUtils.createValidCompactTxnList(txnHandler.getOpenTxnsInfo()); + TxnStore txnHandler = TxnUtils.getTxnStore(conf); + txns = TxnUtils.createValidCompactTxnList(txnHandler.getOpenTxnsInfo()); + } + List compactionCommands = + getCompactionCommands(t, conf, hms, compactionMetaInfo, execute, db, txns); + compactions.addAll(compactionCommands); + } + /*todo: handle renaming files somewhere*/ + } catch (Exception e) { + if (ExceptionUtils.getRootCause(e) instanceof AccessControlException) { + // this could be external table with 0 permission for hive user + LOG.warn("WARNING: Unable to access {}.{}. Skipping table.. If this is an ACID table run " + + "compaction manually.. exception: {}", dbName, tableName, e); + } else { + throw e; + } } - List compactionCommands = - getCompactionCommands(t, conf, hms, compactionMetaInfo, execute, db, txns); - compactions.addAll(compactionCommands); } - /*todo: handle renaming files somewhere*/ + } catch (Exception e) { + if (ExceptionUtils.getRootCause(e) instanceof AccessControlException) { + // we may not have access to read all tables from this db + LOG.warn("WARNING: Unable to access {}. Skipping database.. If this database contains ACID tables run " + + "compaction manually.. exception: {}", dbName, e); + } else { + throw e; + } } } makeCompactionScript(compactions, scriptLocation, compactionMetaInfo); diff --git a/upgrade-acid/pre-upgrade/src/test/java/org/apache/hadoop/hive/upgrade/acid/TestPreUpgradeTool.java b/upgrade-acid/pre-upgrade/src/test/java/org/apache/hadoop/hive/upgrade/acid/TestPreUpgradeTool.java index fe4b08b3f3..c616110c02 100644 --- a/upgrade-acid/pre-upgrade/src/test/java/org/apache/hadoop/hive/upgrade/acid/TestPreUpgradeTool.java +++ b/upgrade-acid/pre-upgrade/src/test/java/org/apache/hadoop/hive/upgrade/acid/TestPreUpgradeTool.java @@ -46,8 +46,13 @@ import org.junit.rules.TestName; import java.io.File; +import java.nio.file.Files; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; import java.util.ArrayList; import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; public class TestPreUpgradeTool { @@ -130,6 +135,81 @@ void onWaitForCompaction() throws MetaException { * */ } + @Test + public void testUpgradeExternalTable() throws Exception { + int[][] data = {{1,2}, {3, 4}, {5, 6}}; + runStatementOnDriver("drop table if exists TExternal"); + + runStatementOnDriver("create table TExternal (a int, b int) stored as orc tblproperties('transactional'='false')"); + + //this needs major compaction + runStatementOnDriver("insert into TExternal" + makeValuesClause(data)); + + String[] args = {"-location", getTestDataDir(), "-execute"}; + PreUpgradeTool.callback = new PreUpgradeTool.Callback() { + @Override + void onWaitForCompaction() throws MetaException { + runWorker(hiveConf); + } + }; + PreUpgradeTool.pollIntervalMs = 1; + PreUpgradeTool.hiveConf = hiveConf; + PreUpgradeTool.main(args); + + TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf); + + ShowCompactResponse resp = txnHandler.showCompact(new ShowCompactRequest()); + Assert.assertEquals(0, resp.getCompactsSize()); + for(ShowCompactResponseElement e : resp.getCompacts()) { + Assert.assertEquals(e.toString(), TxnStore.CLEANING_RESPONSE, e.getState()); + } + + String[] args2 = {"-location", getTestDataDir()}; + PreUpgradeTool.main(args2); + } + + @Test + public void testUpgradeExternalTableNoReadPermissionForUser() throws Exception { + int[][] data = {{1,2}, {3, 4}, {5, 6}}; + runStatementOnDriver("drop table if exists TExternal"); + + runStatementOnDriver("create table TExternal (a int, b int) stored as orc tblproperties('transactional'='false')"); + + //this needs major compaction + runStatementOnDriver("insert into TExternal" + makeValuesClause(data)); + + String tableDir = getWarehouseDir() + "/texternal"; + File tablePath = new File(tableDir); + try { + Set perms = PosixFilePermissions.fromString("-w-------"); + Files.setPosixFilePermissions(tablePath.toPath(), perms); + String[] args = {"-location", getTestDataDir(), "-execute"}; + PreUpgradeTool.callback = new PreUpgradeTool.Callback() { + @Override + void onWaitForCompaction() throws MetaException { + runWorker(hiveConf); + } + }; + PreUpgradeTool.pollIntervalMs = 1; + PreUpgradeTool.hiveConf = hiveConf; + PreUpgradeTool.main(args); + + TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf); + + ShowCompactResponse resp = txnHandler.showCompact(new ShowCompactRequest()); + Assert.assertEquals(0, resp.getCompactsSize()); + for (ShowCompactResponseElement e : resp.getCompacts()) { + Assert.assertEquals(e.toString(), TxnStore.CLEANING_RESPONSE, e.getState()); + } + + String[] args2 = {"-location", getTestDataDir()}; + PreUpgradeTool.main(args2); + } finally { + Set perms = PosixFilePermissions.fromString("rwxrw----"); + Files.setPosixFilePermissions(tablePath.toPath(), perms); + } + } + private static void runWorker(HiveConf hiveConf) throws MetaException { AtomicBoolean stop = new AtomicBoolean(true); Worker t = new Worker(); @@ -203,6 +283,12 @@ private void setUpInternal() throws Exception { hiveConf .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); + hiveConf + .setVar(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS, + "org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener"); + hiveConf + .setVar(HiveConf.ConfVars.HIVE_METASTORE_AUTHORIZATION_MANAGER, + "org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider"); hiveConf.setBoolVar(HiveConf.ConfVars.MERGE_CARDINALITY_VIOLATION_CHECK, true); hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false); TxnDbUtil.setConfValues(hiveConf);