diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java index 84699ce192..e3bdc8d7b7 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java @@ -108,22 +108,21 @@ public void testCreateFunctionIncrementalReplication() throws Throwable { .run("CREATE FUNCTION " + primaryDbName + ".testFunctionTwo as 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFMax'"); + //only testFunctionOne should be replicated, functions created without 'using' clause not supported WarehouseInstance.Tuple incrementalDump = primary.dump(primaryDbName); replica.load(replicatedDbName, primaryDbName) .run("REPL STATUS " + replicatedDbName) .verifyResult(incrementalDump.lastReplicationId) .run("SHOW FUNCTIONS LIKE '" + replicatedDbName + "%'") - .verifyResults(new String[] { replicatedDbName + ".testFunctionOne", - replicatedDbName + ".testFunctionTwo" }); + .verifyResults(new String[] { replicatedDbName + ".testFunctionOne"}); // Test the idempotent behavior of CREATE FUNCTION replica.load(replicatedDbName, primaryDbName) .run("REPL STATUS " + replicatedDbName) .verifyResult(incrementalDump.lastReplicationId) .run("SHOW FUNCTIONS LIKE '" + replicatedDbName + "%'") - .verifyResults(new String[] { replicatedDbName + ".testFunctionOne", - replicatedDbName + ".testFunctionTwo" }); + .verifyResults(new String[] { replicatedDbName + ".testFunctionOne"}); } @Test diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateFunctionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateFunctionHandler.java index b5a910f26a..59eb0f552e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateFunctionHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateFunctionHandler.java @@ -22,6 +22,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.ReplChangeManager; +import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NotificationEvent; import org.apache.hadoop.hive.metastore.messaging.CreateFunctionMessage; @@ -51,6 +52,12 @@ CreateFunctionMessage eventMessage(String stringRepresentation) { @Override public void handle(Context withinContext) throws Exception { + Function functionObj = eventMessage.getFunctionObj(); + if (functionObj.getResourceUris() == null || functionObj.getResourceUris().isEmpty()) { + LOG.info("Not replicating function: " + functionObj.getFunctionName() + " as it seems to have been created " + + "without USING clause"); + return; + } LOG.info("Processing#{} CREATE_FUNCTION message : {}", fromEventId(), eventMessageAsJSON); Path metadataPath = new Path(withinContext.eventRoot, EximUtil.METADATA_NAME); Path dataPath = new Path(withinContext.eventRoot, EximUtil.DATA_PATH_NAME); @@ -58,7 +65,7 @@ public void handle(Context withinContext) throws Exception { boolean copyAtLoad = withinContext.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET); List functionBinaryCopyPaths = new ArrayList<>(); try (JsonWriter jsonWriter = new JsonWriter(fileSystem, metadataPath)) { - FunctionSerializer serializer = new FunctionSerializer(eventMessage.getFunctionObj(), + FunctionSerializer serializer = new FunctionSerializer(functionObj, dataPath, copyAtLoad, withinContext.hiveConf); serializer.writeTo(jsonWriter, withinContext.replicationSpec); functionBinaryCopyPaths.addAll(serializer.getFunctionBinaryCopyPaths());