diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java index 862140f0ed..d161841e4f 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java @@ -73,9 +73,9 @@ import org.junit.rules.TestRule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.ErrorMsg; import javax.annotation.Nullable; - import java.io.File; import java.io.FileWriter; import java.io.IOException; @@ -853,7 +853,8 @@ public NotificationEventResponse apply(@Nullable NotificationEventResponse event InjectableBehaviourObjectStore.setGetNextNotificationBehaviour(eventIdSkipper); advanceDumpDir(); - verifyFail("REPL DUMP " + dbName + " FROM " + replDumpId, driver); + CommandProcessorResponse ret = driver.run("REPL DUMP " + dbName + " FROM " + replDumpId); + assertTrue(ret.getResponseCode() == ErrorMsg.REPL_EVENTS_MISSING_IN_METASTORE.getErrorCode()); eventIdSkipper.assertInjectionsPerformed(true,false); InjectableBehaviourObjectStore.resetGetNextNotificationBehaviour(); // reset the behaviour } @@ -3157,6 +3158,32 @@ public void testRecycleFileDropTempTable() throws IOException { assertTrue(fileCount == fileCountAfter); } + @Test + public void testLoadCmPathMissing() throws IOException { + String dbName = createDB(testName.getMethodName(), driver); + run("CREATE TABLE " + dbName + ".normal(a int)", driver); + run("INSERT INTO " + dbName + ".normal values (1)", driver); + + advanceDumpDir(); + run("repl dump " + dbName, true, driver); + String dumpLocation = getResult(0, 0, driver); + + run("DROP TABLE " + dbName + ".normal", driver); + + String cmDir = hconf.getVar(HiveConf.ConfVars.REPLCMDIR); + Path path = new Path(cmDir); + FileSystem fs = path.getFileSystem(hconf); + ContentSummary cs = fs.getContentSummary(path); + long fileCount = cs.getFileCount(); + assertTrue(fileCount != 0); + fs.delete(path); + + CommandProcessorResponse ret = driverMirror.run("REPL LOAD " + dbName + " FROM '" + dumpLocation + "'"); + assertTrue(ret.getResponseCode() == ErrorMsg.REPL_FILE_MISSING_FROM_SRC_AND_CM_PATH.getErrorCode()); + run("drop database " + dbName, true, driver); + fs.create(path, false); + } + @Test public void testDumpNonReplDatabase() throws IOException { String dbName = createDBNonRepl(testName.getMethodName(), driver); diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java index b217259553..f597bef965 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java @@ -45,6 +45,7 @@ import org.junit.rules.ExpectedException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.ErrorMsg; import java.io.ByteArrayInputStream; import java.io.InputStream; @@ -2927,6 +2928,27 @@ public void testCreateTableExecAsync() throws Exception { stmt.close(); } + @Test + public void testReplErrorScenarios() throws Exception { + HiveStatement stmt = (HiveStatement) con.createStatement(); + + try { + // source of replication not set + stmt.execute("repl dump default"); + } catch(SQLException e){ + assertTrue(e.getErrorCode() == ErrorMsg.REPL_DATABASE_IS_NOT_SOURCE_OF_REPLICATION.getErrorCode()); + } + + try { + // invalid load path + stmt.execute("repl load default1 from '/tmp/junk'"); + } catch(SQLException e){ + assertTrue(e.getErrorCode() == ErrorMsg.REPL_LOAD_PATH_NOT_FOUND.getErrorCode()); + } + + stmt.close(); + } + /** * Test {@link HiveStatement#executeAsync(String)} for an insert overwrite into a table * @throws Exception diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 63ce36aabd..693ab25855 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -499,6 +499,15 @@ + " Please fix and try again.", true), SPARK_RUNTIME_OOM(20015, "Spark job failed because of out of memory."), + //if the error message is changed for REPL_EVENTS_MISSING_IN_METASTORE, then need modification in getNextNotification + //method in HiveMetaStoreClient + REPL_EVENTS_MISSING_IN_METASTORE(20016, "Notification events are missing in the meta store."), + REPL_BOOTSTRAP_LOAD_PATH_NOT_VALID(20017, "Target database is bootstrapped from some other path."), + REPL_FILE_MISSING_FROM_SRC_AND_CM_PATH(20018, "File is missing from both source and cm path."), + REPL_LOAD_PATH_NOT_FOUND(20019, "Load path does not exist."), + REPL_DATABASE_IS_NOT_SOURCE_OF_REPLICATION(20020, + "Source of replication (repl.source.for) is not set in the database properties."), + // An exception from runtime that will show the full stack to client UNRESOLVED_RT_EXCEPTION(29999, "Runtime Error: {0}", "58004", true), @@ -586,6 +595,8 @@ SPARK_JOB_INTERRUPTED(30044, "Spark job was interrupted while executing"), + REPL_FILE_SYSTEM_OPERATION_RETRY(30045, "Replication file system operation retry expired."), + //========================== 40000 range starts here ========================// SPARK_JOB_RUNTIME_ERROR(40001, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java index 5fecf81602..4ace861165 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.plan.CopyWork; import org.apache.hadoop.hive.ql.plan.ReplCopyWork; import org.apache.hadoop.hive.ql.parse.repl.CopyUtils; +import org.apache.hadoop.hive.ql.ErrorMsg; import java.io.BufferedReader; import java.io.IOException; @@ -165,7 +166,7 @@ protected int execute(DriverContext driverContext) { } catch (Exception e) { console.printError("Failed with exception " + e.getMessage(), "\n" + StringUtils.stringifyException(e)); - return (1); + return ErrorMsg.getErrorMsg(e.getMessage()).getErrorCode(); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java index ccdf04aae7..78a0ba3708 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java @@ -68,6 +68,7 @@ import org.apache.hadoop.hive.ql.plan.api.StageType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.ErrorMsg; import java.io.Serializable; import java.util.ArrayList; @@ -123,7 +124,7 @@ protected int execute(DriverContext driverContext) { } catch (Exception e) { LOG.error("failed", e); setException(e); - return 1; + return ErrorMsg.getErrorMsg(e.getMessage()).getErrorCode(); } return 0; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadTask.java index 76fb2a3c5a..b33a774307 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadTask.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.parse.repl.ReplLogger; import org.apache.hadoop.hive.ql.plan.api.StageType; +import org.apache.hadoop.hive.ql.ErrorMsg; import java.io.Serializable; import java.util.ArrayList; @@ -223,7 +224,7 @@ a database ( directory ) } catch (Exception e) { LOG.error("failed replication", e); setException(e); - return 1; + return ErrorMsg.getErrorMsg(e.getMessage()).getErrorCode(); } LOG.info("completed load task run : {}", work.executedLoadTask()); return 0; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/DatabaseEventsIterator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/DatabaseEventsIterator.java index ecedf9b1e4..f778cb42f7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/DatabaseEventsIterator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/DatabaseEventsIterator.java @@ -91,8 +91,8 @@ public boolean hasNext() { return true; } catch (Exception e) { // may be do some retry logic here. - throw new RuntimeException("could not traverse the file via remote iterator " + dbLevelPath, - e); + LOG.error("could not traverse the file via remote iterator " + dbLevelPath, e); + throw new RuntimeException(e.getMessage(), e); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java index c3bb1328d3..f37de3e808 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java @@ -61,6 +61,7 @@ import java.util.List; import java.util.Map; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_DBNAME; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_LIMIT; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_REPL_CONFIG; @@ -110,7 +111,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { try { initReplDump(ast); } catch (HiveException e) { - throw new SemanticException("repl dump failed " + e.getMessage()); + throw new SemanticException(e.getMessage(), e); } analyzeReplDump(ast); break; @@ -147,11 +148,8 @@ private void initReplDump(ASTNode ast) throws HiveException { if (null != replConfigs) { for (Map.Entry config : replConfigs.entrySet()) { conf.set(config.getKey(), config.getValue()); - if ("hive.repl.dump.metadata.only".equalsIgnoreCase(config.getKey()) && - "true".equalsIgnoreCase(config.getValue())) { - isMetaDataOnly = true; - } } + isMetaDataOnly = HiveConf.getBoolVar(conf, REPL_DUMP_METADATA_ONLY); } } else if (ast.getChild(currNode).getType() == TOK_TABNAME) { // optional tblName was specified. @@ -185,9 +183,10 @@ private void initReplDump(ASTNode ast) throws HiveException { for (String dbName : Utils.matchesDb(db, dbNameOrPattern)) { Database database = db.getDatabase(dbName); if (database != null) { - if (!ReplChangeManager.isSourceOfReplication(database) && !isMetaDataOnly) { - throw new SemanticException("Cannot dump database " + dbName + - " as it is not a source of replication"); + if (!isMetaDataOnly && !ReplChangeManager.isSourceOfReplication(database)) { + LOG.error("Cannot dump database " + dbName + + " as it is not a source of replication (repl.source.for)"); + throw new SemanticException(ErrorMsg.REPL_DATABASE_IS_NOT_SOURCE_OF_REPLICATION.getMsg()); } } else { throw new SemanticException("Cannot dump database " + dbName + " as it does not exist"); @@ -366,7 +365,8 @@ private void analyzeReplLoad(ASTNode ast) throws SemanticException { if (!fs.exists(loadPath)) { // supposed dump path does not exist. - throw new FileNotFoundException(loadPath.toUri().toString()); + LOG.error("File not found " + loadPath.toUri().toString()); + throw new FileNotFoundException(ErrorMsg.REPL_LOAD_PATH_NOT_FOUND.getMsg()); } // Now, the dumped path can be one of three things: @@ -512,7 +512,7 @@ private void analyzeReplLoad(ASTNode ast) throws SemanticException { } catch (Exception e) { // TODO : simple wrap & rethrow for now, clean up with error codes - throw new SemanticException(e); + throw new SemanticException(e.getMessage(), e); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java index 79b4652404..7e8d520d93 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java @@ -31,6 +31,8 @@ import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.ql.metadata.HiveFatalException; import javax.security.auth.login.LoginException; import java.io.FileNotFoundException; @@ -68,7 +70,7 @@ public CopyUtils(String distCpDoAsUser, HiveConf hiveConf) { // changed/removed during copy, so double check the checksum after copy, // if not match, copy again from cm public void copyAndVerify(FileSystem destinationFs, Path destRoot, - List srcFiles) throws IOException, LoginException { + List srcFiles) throws IOException, LoginException, HiveFatalException { Map>> map = fsToFileMap(srcFiles, destRoot); for (Map.Entry>> entry : map.entrySet()) { FileSystem sourceFs = entry.getKey(); @@ -92,7 +94,7 @@ public void copyAndVerify(FileSystem destinationFs, Path destRoot, private void doCopyRetry(FileSystem sourceFs, List srcFileList, FileSystem destinationFs, Path destination, - boolean useRegularCopy) throws IOException, LoginException { + boolean useRegularCopy) throws IOException, LoginException, HiveFatalException { int repeat = 0; boolean isCopyError = false; List pathList = Lists.transform(srcFileList, ReplChangeManager.FileInfo::getEffectivePath); @@ -145,7 +147,7 @@ private void doCopyRetry(FileSystem sourceFs, List s // If still files remains to be copied due to failure/checksum mismatch after several attempts, then throw error if (!pathList.isEmpty()) { LOG.error("File copy failed even after several attempts. Files list: " + pathList); - throw new IOException("File copy failed even after several attempts."); + throw new IOException(ErrorMsg.REPL_FILE_SYSTEM_OPERATION_RETRY.getMsg()); } } @@ -154,7 +156,7 @@ private void doCopyRetry(FileSystem sourceFs, List s // itself is missing, then throw error. private List getFilesToRetry(FileSystem sourceFs, List srcFileList, FileSystem destinationFs, Path destination, boolean isCopyError) - throws IOException { + throws IOException, HiveFatalException { List pathList = new ArrayList(); // Going through file list and make the retry list @@ -190,9 +192,9 @@ private void doCopyRetry(FileSystem sourceFs, List s srcPath = srcFile.getEffectivePath(); if (null == srcPath) { // This case possible if CM path is not enabled. - LOG.error("File copy failed and likely source file is deleted or modified. " + LOG.error("File copy failed and likely source file is deleted or modified." + "Source File: " + srcFile.getSourcePath()); - throw new IOException("File copy failed and likely source file is deleted or modified."); + throw new HiveFatalException(ErrorMsg.REPL_FILE_MISSING_FROM_SRC_AND_CM_PATH.getMsg()); } if (!srcFile.isUseSourcePath() && !sourceFs.exists(srcFile.getCmPath())) { @@ -201,7 +203,7 @@ private void doCopyRetry(FileSystem sourceFs, List s + "Missing Source File: " + srcFile.getSourcePath() + ", CM File: " + srcFile.getCmPath() + ". " + "Try setting higher value for hive.repl.cm.retain in source warehouse. " + "Also, bootstrap the system again to get back the consistent replicated state."); - throw new IOException("Both source and CM path are missing from source."); + throw new HiveFatalException(ErrorMsg.REPL_FILE_MISSING_FROM_SRC_AND_CM_PATH.getMsg()); } pathList.add(srcPath); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java index 20ff23a46b..b60be887b2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java @@ -162,7 +162,7 @@ private void writeData(PartitionIterable partitions) throws SemanticException { .export(replicationSpec); } } catch (Exception e) { - throw new SemanticException(e); + throw new SemanticException(e.getMessage(), e); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java index 5e88927f9c..070c830f75 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java @@ -21,12 +21,10 @@ import java.io.IOException; import java.io.OutputStreamWriter; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import javax.security.auth.login.LoginException; -import org.apache.curator.shaded.com.google.common.collect.Lists; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -34,6 +32,7 @@ import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.ReplChangeManager; +import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.HiveInputFormat; @@ -163,7 +162,7 @@ private void exportFilesAsList() throws SemanticException, IOException, LoginExc logger.info("writeFilesList failed", e); if (repeat >= FileUtils.MAX_IO_ERROR_RETRY) { logger.error("exporting data files in dir : " + dataPathList + " to " + exportRootDataDir + " failed"); - throw e; + throw new IOException(ErrorMsg.REPL_FILE_SYSTEM_OPERATION_RETRY.getMsg()); } int sleepTime = FileUtils.getSleepTime(repeat - 1); diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 27b303283e..0eb7f1ae07 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -130,6 +130,9 @@ static final protected Logger LOG = LoggerFactory.getLogger(HiveMetaStoreClient.class); + //copied from ErrorMsg.java + private static final String REPL_EVENTS_MISSING_IN_METASTORE = "Notification events are missing in the meta store."; + public HiveMetaStoreClient(Configuration conf) throws MetaException { this(conf, null, true); } @@ -2706,7 +2709,7 @@ public NotificationEventResponse getNextNotification(long lastEventId, int maxEv + "Try setting higher value for hive.metastore.event.db.listener.timetolive. " + "Also, bootstrap the system again to get back the consistent replicated state.", nextEventId, e.getEventId()); - throw new IllegalStateException("Notification events are missing."); + throw new IllegalStateException(REPL_EVENTS_MISSING_IN_METASTORE); } if ((filter != null) && filter.accept(e)) { filtered.addToEvents(e); diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventUtils.java index 7d8c1d49d9..2b16897533 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventUtils.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventUtils.java @@ -93,7 +93,7 @@ public long getDbNotificationEventsCount(long fromEventId, String dbName) throws try { return msc.getNextNotification(pos,getBatchSize(), filter).getEvents(); } catch (TException e) { - throw new IOException(e); + throw new IOException(e.getMessage(), e); } } } @@ -179,7 +179,7 @@ public boolean hasNext() { // but throwing the exception is the appropriate result here, and hasNext() // signature will only allow RuntimeExceptions. Iterator.hasNext() really // should have allowed IOExceptions - throw new RuntimeException(e); + throw new RuntimeException(e.getMessage(), e); } // New batch has been fetched. If it's not empty, we have more elements to process. return !batch.isEmpty();