diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java index 00620528ae..c9d978626e 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java @@ -49,7 +49,6 @@ import org.apache.hadoop.hive.metastore.messaging.event.filters.DatabaseAndTableFilter; import org.apache.hadoop.hive.metastore.messaging.event.filters.EventBoundaryFilter; import org.apache.hadoop.hive.metastore.messaging.event.filters.MessageFormatFilter; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.DriverFactory; import org.apache.hadoop.hive.ql.IDriver; @@ -79,7 +78,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Map; import static junit.framework.Assert.assertFalse; import static junit.framework.Assert.assertTrue; diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java index 2a48527a31..cb2cb4089d 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java @@ -234,11 +234,11 @@ public void testMultipleStagesOfReplicationLoadTask() throws Throwable { WarehouseInstance.Tuple tuple = primary .run("use " + primaryDbName) .run("create table t1 (id int)") - .run("create table t2 (place string) partitioned by (country string)") + .run("create table T2 (place string) partitioned by (country string)") .run("insert into table t2 partition(country='india') values ('bangalore')") .run("insert into table t2 partition(country='us') values ('austin')") .run("insert into table t2 partition(country='france') values ('paris')") - .run("create table t3 (rank int)") + .run("create table T3 (rank int)") .dump(primaryDbName, null); // each table creation itself takes more than one task, give we are giving a max of 1, we should hit multiple runs. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/DBSerializer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/DBSerializer.java index 15b7e138ed..75285c61cf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/DBSerializer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/DBSerializer.java @@ -38,6 +38,7 @@ public DBSerializer(Database dbObject) { @Override public void writeTo(JsonWriter writer, ReplicationSpec additionalPropertiesProvider) throws SemanticException, IOException { + dbObject.setName(dbObject.getName().toLowerCase()); dbObject.putToParameters( ReplicationSpec.KEY.CURR_STATE_ID.toString(), additionalPropertiesProvider.getCurrentReplicationState() diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FunctionSerializer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FunctionSerializer.java index f72f430a09..33f75b34e1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FunctionSerializer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FunctionSerializer.java @@ -63,6 +63,8 @@ public void writeTo(JsonWriter writer, ReplicationSpec additionalPropertiesProvi } } Function copyObj = new Function(this.function); + copyObj.setFunctionName(copyObj.getFunctionName().toLowerCase()); + copyObj.setDbName(copyObj.getDbName().toLowerCase()); if (!resourceUris.isEmpty()) { assert resourceUris.size() == this.function.getResourceUris().size(); copyObj.setResourceUris(resourceUris); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/PartitionSerializer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/PartitionSerializer.java index ce83523389..998c49f7ca 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/PartitionSerializer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/PartitionSerializer.java @@ -55,6 +55,8 @@ public void writeTo(JsonWriter writer, ReplicationSpec additionalPropertiesProvi partition.putToParameters("EXTERNAL", "FALSE"); } } + partition.setDbName(partition.getDbName().toLowerCase()); + partition.setTableName(partition.getTableName().toLowerCase()); writer.jsonGenerator.writeString(serializer.toString(partition, UTF_8)); writer.jsonGenerator.flush(); } catch (TException e) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java index 143808bb85..76939e83b3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java @@ -52,12 +52,14 @@ public void writeTo(JsonWriter writer, ReplicationSpec additionalPropertiesProvi return; } - Table tTable = tableHandle.getTTable(); - tTable = addPropertiesToTable(tTable, additionalPropertiesProvider); + Table table = tableHandle.getTTable(); + table = addPropertiesToTable(table, additionalPropertiesProvider); try { TSerializer serializer = new TSerializer(new TJSONProtocol.Factory()); + table.setDbName(table.getDbName().toLowerCase()); + table.setTableName(table.getTableName().toLowerCase()); writer.jsonGenerator - .writeStringField(FIELD_NAME, serializer.toString(tTable, UTF_8)); + .writeStringField(FIELD_NAME, serializer.toString(table, UTF_8)); writer.jsonGenerator.writeFieldName(PartitionSerializer.FIELD_NAME); writePartitions(writer, additionalPropertiesProvider); } catch (TException e) { diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/dump/io/DBSerializerTest.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/dump/io/DBSerializerTest.java new file mode 100644 index 0000000000..c4080240a6 --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/dump/io/DBSerializerTest.java @@ -0,0 +1,46 @@ +package org.apache.hadoop.hive.ql.parse.repl.dump.io; + +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.ql.parse.ReplicationSpec; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.HashMap; + +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.same; +import static org.powermock.api.mockito.PowerMockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class DBSerializerTest { + @Mock + private FileSystem fs; + @Mock + private Path writePath; + + @Test + public void databaseNameIsInLowercase() throws IOException, SemanticException { + DBSerializer dbSerializer = + new DBSerializer(new Database("DBName", "", "", new HashMap<>())); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + FSDataOutputStream stream = new FSDataOutputStream(out, null); + when(fs.create(same(writePath))).thenReturn(stream); + + try (JsonWriter writer = new JsonWriter(fs, writePath)) { + ReplicationSpec additionalPropertiesProvider = new ReplicationSpec(); + additionalPropertiesProvider.setCurrentReplicationState("34"); + dbSerializer.writeTo(writer, additionalPropertiesProvider); + } + String outputString = out.toString(); + assertTrue(outputString + " does not contain the database name in lowercase", + outputString.contains("dbname")); + } +} \ No newline at end of file diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/dump/io/FunctionSerializerTest.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/dump/io/FunctionSerializerTest.java new file mode 100644 index 0000000000..ef74c0cb8e --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/dump/io/FunctionSerializerTest.java @@ -0,0 +1,54 @@ +package org.apache.hadoop.hive.ql.parse.repl.dump.io; + +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.Function; +import org.apache.hadoop.hive.metastore.api.FunctionType; +import org.apache.hadoop.hive.metastore.api.PrincipalType; +import org.apache.hadoop.hive.ql.parse.ReplicationSpec; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Collections; + +import static org.junit.Assert.*; +import static org.mockito.Matchers.same; +import static org.powermock.api.mockito.PowerMockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class FunctionSerializerTest { + @Mock + private FileSystem fs; + @Mock + private Path writePath; + + @Test + public void databaseNameAndFunctionNameAreInLowerCase() throws IOException, + SemanticException { + Function function = new Function("TESTFUNCTION", "DBNAME", + "org.apache.some.class.Clazz", "test", + PrincipalType.USER, Integer.MAX_VALUE, FunctionType.JAVA, Collections.emptyList()); + FunctionSerializer functionSerializer = new FunctionSerializer(function, new HiveConf()); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + FSDataOutputStream stream = new FSDataOutputStream(out, null); + when(fs.create(same(writePath))).thenReturn(stream); + + try (JsonWriter writer = new JsonWriter(fs, writePath)) { + functionSerializer.writeTo(writer, new ReplicationSpec()); + } + + String outputString = out.toString(); + assertTrue(outputString + " does not contain the function name in lowercase", + outputString.contains("testfunction")); + assertTrue(outputString + " does not contain the database name in lowercase", + outputString.contains("dbname")); + } +} \ No newline at end of file diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/dump/io/PartitionSerializerTest.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/dump/io/PartitionSerializerTest.java new file mode 100644 index 0000000000..453247d060 --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/dump/io/PartitionSerializerTest.java @@ -0,0 +1,51 @@ +package org.apache.hadoop.hive.ql.parse.repl.dump.io; + +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.ql.parse.ReplicationSpec; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Collections; + +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.same; +import static org.powermock.api.mockito.PowerMockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class PartitionSerializerTest { + @Mock + private FileSystem fs; + @Mock + private Path writePath; + + @Test + public void tableNameAndDatabaseNameAreInLowerCase() throws IOException, SemanticException { + Partition partition = new Partition(Collections.emptyList(), "DBName", "TABLENAME", + Integer.MAX_VALUE, Integer.MAX_VALUE, new StorageDescriptor(), Collections.emptyMap()); + PartitionSerializer partitionSerializer = new PartitionSerializer(partition); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + FSDataOutputStream stream = new FSDataOutputStream(out, null); + when(fs.create(same(writePath))).thenReturn(stream); + + try (JsonWriter writer = new JsonWriter(fs, writePath)) { + writer.jsonGenerator.writeFieldName("PartitionField"); + partitionSerializer.writeTo(writer, new ReplicationSpec()); + } + + String outputString = out.toString(); + assertTrue(outputString + " does not contain the table name in lowercase", + outputString.contains("tablename")); + assertTrue(outputString + " does not contain the database name in lowercase", + outputString.contains("dbname")); + } +} \ No newline at end of file diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializerTest.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializerTest.java new file mode 100644 index 0000000000..72d10ebaf2 --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializerTest.java @@ -0,0 +1,57 @@ +package org.apache.hadoop.hive.ql.parse.repl.dump.io; + +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.ReplicationSpec; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Collections; + +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.same; +import static org.powermock.api.mockito.PowerMockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class TableSerializerTest { + @Mock + private FileSystem fs; + @Mock + private Path writePath; + + @Test + public void testTableNameAndDatabaseNameIsInLowerCase() throws IOException, SemanticException { + Table tableHandle = new Table(); + tableHandle.setTTable( + new org.apache.hadoop.hive.metastore.api.Table("TableName", "dbName", "test", + Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE, new StorageDescriptor(), + Collections.emptyList(), Collections.emptyMap(), "", "", + "")); + + TableSerializer tableSerializer = + new TableSerializer(tableHandle, null, new HiveConf()); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + FSDataOutputStream stream = + new FSDataOutputStream(out, null); + when(fs.create(same(writePath))).thenReturn(stream); + + try (JsonWriter writer = new JsonWriter(fs, writePath)) { + tableSerializer.writeTo(writer, new ReplicationSpec()); + } + String outputString = out.toString(); + assertTrue(outputString + " does not contain the table name in lowercase", + outputString.contains("tablename")); + assertTrue(outputString + " does not contain the database name in lowercase", + outputString.contains("dbname")); + } +} \ No newline at end of file