diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java index e0547d4cce..748a0ef25f 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java @@ -154,6 +154,32 @@ private void initialize(String cmRoot, String externalTableWarehouseRoot, String MetaStoreTestUtils.startMetaStoreWithRetry(hiveConf, true); + // Add the below mentioned dependency in metastore/pom.xml file. For postgres need to copy postgresql-42.2.1.jar to + // .m2//repository/postgresql/postgresql/9.3-1102.jdbc41/postgresql-9.3-1102.jdbc41.jar. + /* + + mysql + mysql-connector-java + 8.0.15 + + + + postgresql + postgresql + 9.3-1102.jdbc41 + + */ + + /*hiveConf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, "jdbc:mysql://localhost:3306/APP"); + hiveConf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER, "com.mysql.jdbc.Driver"); + hiveConf.setVar(HiveConf.ConfVars.METASTOREPWD, "hivepassword"); + hiveConf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME, "hiveuser");*/ + + /*hiveConf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY,"jdbc:postgresql://localhost/app"); + hiveConf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER, "org.postgresql.Driver"); + hiveConf.setVar(HiveConf.ConfVars.METASTOREPWD, "password"); + hiveConf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME, "postgres");*/ + driver = DriverFactory.newDriver(hiveConf); SessionState.start(new CliSessionState(hiveConf)); client = new HiveMetaStoreClient(hiveConf); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index cb7fdf73b5..b02cdf8a61 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -4712,6 +4712,7 @@ private int createTable(Hive db, CreateTableDesc crtTbl) throws HiveException { if (existingTable != null){ if (crtTbl.getReplicationSpec().allowEventReplacementInto(existingTable.getParameters())){ crtTbl.setReplaceMode(true); // we replace existing table. + ReplicationSpec.copyLastReplId(existingTable.getParameters(), tbl.getParameters()); } else { LOG.debug("DDLTask: Create Table is skipped as table {} is newer than update", crtTbl.getTableName()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 9ab3a9ecb6..14f6fd028f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -132,6 +132,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.LoadTableDesc.LoadFileType; +import org.apache.hadoop.hive.ql.parse.ReplicationSpec; import org.apache.hadoop.hive.ql.session.CreateTableAutomaticGrant; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde2.Deserializer; @@ -2973,6 +2974,7 @@ public Partition createPartition(Table tbl, Map partSpec) throws org.apache.hadoop.hive.metastore.api.Partition ptn = getMSC().getPartition(addPartitionDesc.getDbName(), addPartitionDesc.getTableName(), p.getValues()); if (addPartitionDesc.getReplicationSpec().allowReplacementInto(ptn.getParameters())){ + ReplicationSpec.copyLastReplId(ptn.getParameters(), p.getParameters()); partsToAlter.add(p); } // else ptn already exists, but we do nothing with it. } catch (NoSuchObjectException nsoe){ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java index b087831d04..d55ee208cc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java @@ -419,4 +419,11 @@ public boolean isMigratingToExternalTable() { public void setMigratingToExternalTable() { isMigratingToExternalTable = true; } + + public static void copyLastReplId(Map srcParameter, Map destParameter) { + String lastReplId = srcParameter.get(ReplicationSpec.KEY.CURR_STATE_ID.toString()); + if (lastReplId != null) { + destParameter.put(ReplicationSpec.KEY.CURR_STATE_ID.toString(), lastReplId); + } + } } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index 6df7680a7c..fd85af90f9 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -897,13 +897,13 @@ private void updateReplId(Connection dbConn, ReplLastIdInfo replLastIdInfo) thro if (needUpdateDBReplId) { // not used select for update as it will be updated by single thread only from repl load - rs = stmt.executeQuery("select PARAM_VALUE from DATABASE_PARAMS where PARAM_KEY = " + - "'repl.last.id' and DB_ID = " + dbId); + rs = stmt.executeQuery("select \"PARAM_VALUE\" from \"DATABASE_PARAMS\" where \"PARAM_KEY\" = " + + "'repl.last.id' and \"DB_ID\" = " + dbId); if (!rs.next()) { - query = "insert into DATABASE_PARAMS values ( " + dbId + " , 'repl.last.id' , ? )"; + query = "insert into \"DATABASE_PARAMS\" values ( " + dbId + " , 'repl.last.id' , ? )"; } else { - query = "update DATABASE_PARAMS set PARAM_VALUE = ? where DB_ID = " + dbId + - " and PARAM_KEY = 'repl.last.id'"; + query = "update \"DATABASE_PARAMS\" set \"PARAM_VALUE\" = ? where \"DB_ID\" = " + dbId + + " and \"PARAM_KEY\" = 'repl.last.id'"; } close(rs); params = Arrays.asList(lastReplId); @@ -935,13 +935,13 @@ private void updateReplId(Connection dbConn, ReplLastIdInfo replLastIdInfo) thro pst.close(); // select for update is not required as only one task will update this during repl load. - rs = stmt.executeQuery("select PARAM_VALUE from TABLE_PARAMS where PARAM_KEY = " + - "'repl.last.id' and TBL_ID = " + tblId); + rs = stmt.executeQuery("select \"PARAM_VALUE\" from \"TABLE_PARAMS\" where \"PARAM_KEY\" = " + + "'repl.last.id' and \"TBL_ID\" = " + tblId); if (!rs.next()) { - query = "insert into TABLE_PARAMS values ( " + tblId + " , 'repl.last.id' , ? )"; + query = "insert into \"TABLE_PARAMS\" values ( " + tblId + " , 'repl.last.id' , ? )"; } else { - query = "update TABLE_PARAMS set PARAM_VALUE = ? where TBL_ID = " + tblId + - " and PARAM_KEY = 'repl.last.id'"; + query = "update \"TABLE_PARAMS\" set \"PARAM_VALUE\" = ? where \"TBL_ID\" = " + tblId + + " and \"PARAM_KEY\" = 'repl.last.id'"; } rs.close(); @@ -988,13 +988,13 @@ private void updateReplId(Connection dbConn, ReplLastIdInfo replLastIdInfo) thro prs = pst.executeQuery(); while (prs.next()) { long partId = prs.getLong(1); - rs = stmt.executeQuery("select PARAM_VALUE from PARTITION_PARAMS where PARAM_KEY " + - " = 'repl.last.id' and PART_ID = " + partId); + rs = stmt.executeQuery("select \"PARAM_VALUE\" from \"PARTITION_PARAMS\" where \"PARAM_KEY\" " + + " = 'repl.last.id' and \"PART_ID\" = " + partId); if (!rs.next()) { - query = "insert into PARTITION_PARAMS values ( " + partId + " , 'repl.last.id' , ? )"; + query = "insert into \"PARTITION_PARAMS\" values ( " + partId + " , 'repl.last.id' , ? )"; } else { - query = "update PARTITION_PARAMS set PARAM_VALUE = ? " + - " where PART_ID = " + partId + " and PARAM_KEY = 'repl.last.id'"; + query = "update \"PARTITION_PARAMS\" set \"PARAM_VALUE\" = ? " + + " where \"PART_ID\" = " + partId + " and \"PARAM_KEY\" = 'repl.last.id'"; } rs.close();