diff --git a/upgrade-acid/pre-upgrade/src/main/java/org/apache/hadoop/hive/upgrade/acid/PreUpgradeTool.java b/upgrade-acid/pre-upgrade/src/main/java/org/apache/hadoop/hive/upgrade/acid/PreUpgradeTool.java index 04782a62ad..5731139ab0 100644 --- a/upgrade-acid/pre-upgrade/src/main/java/org/apache/hadoop/hive/upgrade/acid/PreUpgradeTool.java +++ b/upgrade-acid/pre-upgrade/src/main/java/org/apache/hadoop/hive/upgrade/acid/PreUpgradeTool.java @@ -25,6 +25,7 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -54,6 +55,7 @@ import org.apache.hadoop.hive.ql.io.orc.Reader; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.shims.HadoopShims; +import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hive.common.util.HiveVersionInfo; import org.apache.thrift.TException; @@ -223,6 +225,7 @@ private void prepareAcidUpgradeInternal(String scriptLocation, boolean execute) boolean isAcidEnabled = isAcidEnabled(conf); IMetaStoreClient hms = getHMS(conf); LOG.debug("Looking for databases"); + // if we cannot databases we cannot proceed with pre-upgrade so we don't handle exception here List databases = hms.getAllDatabases();//TException LOG.debug("Found " + databases.size() + " databases to process"); List compactions = new ArrayList<>(); @@ -234,26 +237,46 @@ private void prepareAcidUpgradeInternal(String scriptLocation, boolean execute) } for(String dbName : databases) { - List tables = hms.getAllTables(dbName); - LOG.debug("found " + tables.size() + " tables in " + dbName); - for(String tableName : tables) { - Table t = hms.getTable(dbName, tableName); - LOG.debug("processing table " + Warehouse.getQualifiedName(t)); - if(isAcidEnabled) { - //if acid is off, there can't be any acid tables - nothing to compact - if(txns == null) { + try { + List tables = hms.getAllTables(dbName); + LOG.debug("found " + tables.size() + " tables in " + dbName); + for (String tableName : tables) { + try { + Table t = hms.getTable(dbName, tableName); + LOG.debug("processing table " + Warehouse.getQualifiedName(t)); + if (isAcidEnabled) { + //if acid is off, there can't be any acid tables - nothing to compact + if (txns == null) { /* This API changed from 2.x to 3.0. so this won't even compile with 3.0 but it doesn't need to since we only run this preUpgrade */ - TxnStore txnHandler = TxnUtils.getTxnStore(conf); - txns = TxnUtils.createValidCompactTxnList(txnHandler.getOpenTxnsInfo()); + TxnStore txnHandler = TxnUtils.getTxnStore(conf); + txns = TxnUtils.createValidCompactTxnList(txnHandler.getOpenTxnsInfo()); + } + List compactionCommands = + getCompactionCommands(t, conf, hms, compactionMetaInfo, execute, db, txns); + compactions.addAll(compactionCommands); + } + /*todo: handle renaming files somewhere*/ + } catch (Exception e) { + if (ExceptionUtils.getRootCause(e) instanceof AccessControlException) { + // this could be external table with 0 permission for hive user + LOG.warn("WARNING: Unable to access {}.{}. Skipping table.. If this is an ACID table run " + + "compaction manually.. exception: {}", dbName, tableName, e); + } else { + throw e; + } } - List compactionCommands = - getCompactionCommands(t, conf, hms, compactionMetaInfo, execute, db, txns); - compactions.addAll(compactionCommands); } - /*todo: handle renaming files somewhere*/ + } catch (Exception e) { + if (ExceptionUtils.getRootCause(e) instanceof AccessControlException) { + // we may not have access to read all tables from this db + LOG.warn("WARNING: Unable to access {}. Skipping database.. If this database contains ACID tables run " + + "compaction manually.. exception: {}", dbName, e); + } else { + throw e; + } } } makeCompactionScript(compactions, scriptLocation, compactionMetaInfo); diff --git a/upgrade-acid/pre-upgrade/src/test/java/org/apache/hadoop/hive/upgrade/acid/TestPreUpgradeTool.java b/upgrade-acid/pre-upgrade/src/test/java/org/apache/hadoop/hive/upgrade/acid/TestPreUpgradeTool.java index fe4b08b3f3..5dd0da1111 100644 --- a/upgrade-acid/pre-upgrade/src/test/java/org/apache/hadoop/hive/upgrade/acid/TestPreUpgradeTool.java +++ b/upgrade-acid/pre-upgrade/src/test/java/org/apache/hadoop/hive/upgrade/acid/TestPreUpgradeTool.java @@ -46,8 +46,13 @@ import org.junit.rules.TestName; import java.io.File; +import java.nio.file.Files; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; import java.util.ArrayList; import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; public class TestPreUpgradeTool { @@ -72,31 +77,84 @@ public void testUpgrade() throws Exception { runStatementOnDriver("drop table if exists TFlat"); runStatementOnDriver("drop table if exists TFlatText"); - runStatementOnDriver("create table TAcid (a int, b int) clustered by (b) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); - runStatementOnDriver("create table TAcidPart (a int, b int) partitioned by (p tinyint) clustered by (b) into 2 buckets stored" + - " as orc TBLPROPERTIES ('transactional'='true')"); - //on 2.x these are guaranteed to not be acid - runStatementOnDriver("create table TFlat (a int, b int) stored as orc tblproperties('transactional'='false')"); - runStatementOnDriver("create table TFlatText (a int, b int) stored as textfile tblproperties('transactional'='false')"); + try { + runStatementOnDriver( + "create table TAcid (a int, b int) clustered by (b) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); + runStatementOnDriver( + "create table TAcidPart (a int, b int) partitioned by (p tinyint) clustered by (b) into 2 buckets stored" + + " as orc TBLPROPERTIES ('transactional'='true')"); + //on 2.x these are guaranteed to not be acid + runStatementOnDriver("create table TFlat (a int, b int) stored as orc tblproperties('transactional'='false')"); + runStatementOnDriver( + "create table TFlatText (a int, b int) stored as textfile tblproperties('transactional'='false')"); - //this needs major compaction - runStatementOnDriver("insert into TAcid" + makeValuesClause(data)); - runStatementOnDriver("update TAcid set a = 1 where b = 2"); + //this needs major compaction + runStatementOnDriver("insert into TAcid" + makeValuesClause(data)); + runStatementOnDriver("update TAcid set a = 1 where b = 2"); + + //this table needs to be converted to CRUD Acid + runStatementOnDriver("insert into TFlat" + makeValuesClause(data)); + + //this table needs to be converted to MM + runStatementOnDriver("insert into TFlatText" + makeValuesClause(data)); - //this table needs to be converted to CRUD Acid - runStatementOnDriver("insert into TFlat" + makeValuesClause(data)); + //p=10 needs major compaction + runStatementOnDriver("insert into TAcidPart partition(p)" + makeValuesClause(dataPart)); + runStatementOnDriver("update TAcidPart set a = 1 where b = 2 and p = 10"); - //this table needs to be converted to MM - runStatementOnDriver("insert into TFlatText" + makeValuesClause(data)); + //todo: add partitioned table that needs conversion to MM/Acid + + //todo: rename files case + String[] args = {"-location", getTestDataDir(), "-execute"}; + PreUpgradeTool.callback = new PreUpgradeTool.Callback() { + @Override + void onWaitForCompaction() throws MetaException { + runWorker(hiveConf); + } + }; + PreUpgradeTool.pollIntervalMs = 1; + PreUpgradeTool.hiveConf = hiveConf; + PreUpgradeTool.main(args); + /* + todo: parse + target/tmp/org.apache.hadoop.hive.upgrade.acid.TestPreUpgradeTool-1527286256834/compacts_1527286277624.sql + make sure it's the only 'compacts' file and contains + ALTER TABLE default.tacid COMPACT 'major'; +ALTER TABLE default.tacidpart PARTITION(p=10Y) COMPACT 'major'; + * */ - //p=10 needs major compaction - runStatementOnDriver("insert into TAcidPart partition(p)" + makeValuesClause(dataPart)); - runStatementOnDriver("update TAcidPart set a = 1 where b = 2 and p = 10"); + TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf); - //todo: add partitioned table that needs conversion to MM/Acid + ShowCompactResponse resp = txnHandler.showCompact(new ShowCompactRequest()); + Assert.assertEquals(2, resp.getCompactsSize()); + for (ShowCompactResponseElement e : resp.getCompacts()) { + Assert.assertEquals(e.toString(), TxnStore.CLEANING_RESPONSE, e.getState()); + } + + String[] args2 = {"-location", getTestDataDir()}; + PreUpgradeTool.main(args2); + /* + * todo: parse compacts script - make sure there is nothing in it + * */ + } finally { + runStatementOnDriver("drop table if exists TAcid"); + runStatementOnDriver("drop table if exists TAcidPart"); + runStatementOnDriver("drop table if exists TFlat"); + runStatementOnDriver("drop table if exists TFlatText"); + } + } + + @Test + public void testUpgradeExternalTable() throws Exception { + int[][] data = {{1,2}, {3, 4}, {5, 6}}; + runStatementOnDriver("drop table if exists TExternal"); + + runStatementOnDriver("create table TExternal (a int, b int) stored as orc tblproperties('transactional'='false')"); + + //this needs major compaction + runStatementOnDriver("insert into TExternal" + makeValuesClause(data)); - //todo: rename files case String[] args = {"-location", getTestDataDir(), "-execute"}; PreUpgradeTool.callback = new PreUpgradeTool.Callback() { @Override @@ -107,27 +165,59 @@ void onWaitForCompaction() throws MetaException { PreUpgradeTool.pollIntervalMs = 1; PreUpgradeTool.hiveConf = hiveConf; PreUpgradeTool.main(args); - /* - todo: parse - target/tmp/org.apache.hadoop.hive.upgrade.acid.TestPreUpgradeTool-1527286256834/compacts_1527286277624.sql - make sure it's the only 'compacts' file and contains - ALTER TABLE default.tacid COMPACT 'major'; -ALTER TABLE default.tacidpart PARTITION(p=10Y) COMPACT 'major'; - * */ TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf); ShowCompactResponse resp = txnHandler.showCompact(new ShowCompactRequest()); - Assert.assertEquals(2, resp.getCompactsSize()); + Assert.assertEquals(0, resp.getCompactsSize()); for(ShowCompactResponseElement e : resp.getCompacts()) { Assert.assertEquals(e.toString(), TxnStore.CLEANING_RESPONSE, e.getState()); } String[] args2 = {"-location", getTestDataDir()}; PreUpgradeTool.main(args2); - /* - * todo: parse compacts script - make sure there is nothing in it - * */ + } + + @Test + public void testUpgradeExternalTableNoReadPermissionForUser() throws Exception { + int[][] data = {{1,2}, {3, 4}, {5, 6}}; + runStatementOnDriver("drop table if exists TExternal"); + + runStatementOnDriver("create table TExternal (a int, b int) stored as orc tblproperties('transactional'='false')"); + + //this needs major compaction + runStatementOnDriver("insert into TExternal" + makeValuesClause(data)); + + String tableDir = getWarehouseDir() + "/texternal"; + File tablePath = new File(tableDir); + try { + Set perms = PosixFilePermissions.fromString("-w-------"); + Files.setPosixFilePermissions(tablePath.toPath(), perms); + String[] args = {"-location", getTestDataDir(), "-execute"}; + PreUpgradeTool.callback = new PreUpgradeTool.Callback() { + @Override + void onWaitForCompaction() throws MetaException { + runWorker(hiveConf); + } + }; + PreUpgradeTool.pollIntervalMs = 1; + PreUpgradeTool.hiveConf = hiveConf; + PreUpgradeTool.main(args); + + TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf); + + ShowCompactResponse resp = txnHandler.showCompact(new ShowCompactRequest()); + Assert.assertEquals(0, resp.getCompactsSize()); + for (ShowCompactResponseElement e : resp.getCompacts()) { + Assert.assertEquals(e.toString(), TxnStore.CLEANING_RESPONSE, e.getState()); + } + + String[] args2 = {"-location", getTestDataDir()}; + PreUpgradeTool.main(args2); + } finally { + Set perms = PosixFilePermissions.fromString("rwxrw----"); + Files.setPosixFilePermissions(tablePath.toPath(), perms); + } } private static void runWorker(HiveConf hiveConf) throws MetaException { @@ -203,6 +293,12 @@ private void setUpInternal() throws Exception { hiveConf .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); + hiveConf + .setVar(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS, + "org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener"); + hiveConf + .setVar(HiveConf.ConfVars.HIVE_METASTORE_AUTHORIZATION_MANAGER, + "org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider"); hiveConf.setBoolVar(HiveConf.ConfVars.MERGE_CARDINALITY_VIOLATION_CHECK, true); hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false); TxnDbUtil.setConfValues(hiveConf);