diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java index 22fdb69..569bfd0 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java @@ -23,6 +23,7 @@ import java.io.PrintStream; import java.util.ArrayList; import java.util.HashMap; +import java.util.Map; import junit.framework.TestCase; @@ -102,6 +103,9 @@ protected void setUp() throws Exception { Table tbl = new Table(); tbl.setDbName(dbName); tbl.setTableName(tblName); + Map parameters = new HashMap<>(); + parameters.put(AvroSerdeUtils.SCHEMA_URL, avroUri); + tbl.setParameters(parameters); StorageDescriptor sd = new StorageDescriptor(); tbl.setSd(sd); sd.setCols(typ1.getFields()); @@ -206,10 +210,12 @@ public void testExecuteJDOQL() throws Exception { } public void testUpdateFSRootLocation() throws Exception { - redirectOutputStream(); String oldLocationUri = "hdfs://nn.example.com/"; String newLocationUri = "hdfs://nn-ha-uri/"; + String oldSchemaUri = "hdfs://nn.example.com/warehouse/hive/ab.avsc"; + String newSchemaUri = "hdfs://nn-ha-uri/warehouse/hive/ab.avsc"; + String[] args = new String[5]; args[0] = new String("-updateLocation"); args[1] = new String(newLocationUri); @@ -218,18 +224,22 @@ public void testUpdateFSRootLocation() throws Exception { args[4] = new String("avro.schema.url"); try { + checkAvroSchemaURLProps(client.getTable(dbName, tblName), oldSchemaUri); + // perform HA upgrade HiveMetaTool.main(args); String out = os.toString(); boolean b = out.contains(newLocationUri); restoreOutputStream(); assertTrue(b); + checkAvroSchemaURLProps(client.getTable(dbName,tblName), newSchemaUri); //restore the original HDFS root args[1] = new String(oldLocationUri); args[2] = new String(newLocationUri); redirectOutputStream(); HiveMetaTool.main(args); + checkAvroSchemaURLProps(client.getTable(dbName,tblName), oldSchemaUri); restoreOutputStream(); } finally { restoreOutputStream(); @@ -237,6 +247,11 @@ public void testUpdateFSRootLocation() throws Exception { } } + private void checkAvroSchemaURLProps(Table table, String expectedURL) { + assertEquals(expectedURL, table.getParameters().get(AvroSerdeUtils.SCHEMA_URL)); + assertEquals(expectedURL, table.getSd().getParameters().get(AvroSerdeUtils.SCHEMA_URL)); + } + @Override protected void tearDown() throws Exception { try { diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index a8c02b4..583e23f 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -5596,12 +5596,11 @@ public UpdateMDatabaseURIRetVal updateMDatabaseURI(URI oldLoc, URI newLoc, boole } } - public class UpdateMStorageDescriptorTblPropURIRetVal { + public class UpdatePropURIRetVal { private List badRecords; private Map updateLocations; - UpdateMStorageDescriptorTblPropURIRetVal(List badRecords, - Map updateLocations) { + UpdatePropURIRetVal(List badRecords, Map updateLocations) { this.badRecords = badRecords; this.updateLocations = updateLocations; } @@ -5623,6 +5622,72 @@ public void setUpdateLocations(Map updateLocations) { } } + private void updatePropURIHelper(URI oldLoc, URI newLoc, String tblPropKey, boolean isDryRun, + List badRecords, Map updateLocations, + Map parameters) { + URI tablePropLocationURI = null; + if (parameters.containsKey(tblPropKey)) { + String tablePropLocation = parameters.get(tblPropKey); + try { + tablePropLocationURI = new URI(tablePropLocation); + } catch (URISyntaxException e) { + badRecords.add(tablePropLocation); + } catch (NullPointerException e) { + badRecords.add(tablePropLocation); + } + // if tablePropKey that was passed in lead to a valid URI resolution, update it if + //parts of it match the old-NN-loc, else add to badRecords + if (tablePropLocationURI == null) { + badRecords.add(tablePropLocation); + } else { + if (shouldUpdateURI(tablePropLocationURI, oldLoc)) { + String tblPropLoc = parameters.get(tblPropKey).replaceAll(oldLoc.toString(), newLoc + .toString()); + updateLocations.put(tablePropLocationURI.toString(), tblPropLoc); + if (!isDryRun) { + parameters.put(tblPropKey, tblPropLoc); + } + } + } + } + } + + /** The following APIs + * + * - updateMStorageDescriptorTblPropURI + * + * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift. + * + */ + public UpdatePropURIRetVal updateTblPropURI(URI oldLoc, URI newLoc, String tblPropKey, boolean + isDryRun) { + boolean committed = false; + Map updateLocations = new HashMap<>(); + List badRecords = new ArrayList<>(); + UpdatePropURIRetVal retVal = null; + + try { + openTransaction(); + Query query = pm.newQuery(MTable.class); + List mTbls = (List) query.execute(); + pm.retrieveAll(mTbls); + + for (MTable mTbl : mTbls) { + updatePropURIHelper(oldLoc, newLoc, tblPropKey, isDryRun, badRecords, updateLocations, + mTbl.getParameters()); + } + committed = commitTransaction(); + if (committed) { + retVal = new UpdatePropURIRetVal(badRecords, updateLocations); + } + return retVal; + } finally { + if (!committed) { + rollbackTransaction(); + } + } + } + /** The following APIs * * - updateMStorageDescriptorTblPropURI @@ -5630,12 +5695,13 @@ public void setUpdateLocations(Map updateLocations) { * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift. * */ - public UpdateMStorageDescriptorTblPropURIRetVal updateMStorageDescriptorTblPropURI(URI oldLoc, + @Deprecated + public UpdatePropURIRetVal updateMStorageDescriptorTblPropURI(URI oldLoc, URI newLoc, String tblPropKey, boolean isDryRun) { boolean committed = false; Map updateLocations = new HashMap(); List badRecords = new ArrayList(); - UpdateMStorageDescriptorTblPropURIRetVal retVal = null; + UpdatePropURIRetVal retVal = null; try { openTransaction(); @@ -5644,35 +5710,12 @@ public UpdateMStorageDescriptorTblPropURIRetVal updateMStorageDescriptorTblPropU pm.retrieveAll(mSDSs); for(MStorageDescriptor mSDS:mSDSs) { - URI tablePropLocationURI = null; - if (mSDS.getParameters().containsKey(tblPropKey)) { - String tablePropLocation = mSDS.getParameters().get(tblPropKey); - try { - tablePropLocationURI = new URI(tablePropLocation); - } catch (URISyntaxException e) { - badRecords.add(tablePropLocation); - } catch (NullPointerException e) { - badRecords.add(tablePropLocation); - } - // if tablePropKey that was passed in lead to a valid URI resolution, update it if - //parts of it match the old-NN-loc, else add to badRecords - if (tablePropLocationURI == null) { - badRecords.add(tablePropLocation); - } else { - if (shouldUpdateURI(tablePropLocationURI, oldLoc)) { - String tblPropLoc = mSDS.getParameters().get(tblPropKey).replaceAll(oldLoc.toString(), - newLoc.toString()); - updateLocations.put(tablePropLocationURI.toString(), tblPropLoc); - if (!isDryRun) { - mSDS.getParameters().put(tblPropKey, tblPropLoc); - } - } - } - } + updatePropURIHelper(oldLoc, newLoc, tblPropKey, isDryRun, badRecords, updateLocations, + mSDS.getParameters()); } committed = commitTransaction(); if (committed) { - retVal = new UpdateMStorageDescriptorTblPropURIRetVal(badRecords, updateLocations); + retVal = new UpdatePropURIRetVal(badRecords, updateLocations); } return retVal; } finally { diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java b/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java index 7dc5f44..d0ff329 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java @@ -253,15 +253,12 @@ private void printDatabaseURIUpdateSummary(ObjectStore.UpdateMDatabaseURIRetVal } } - private void printTblPropURIUpdateSummary( - ObjectStore.UpdateMStorageDescriptorTblPropURIRetVal retVal, String tablePropKey, - boolean isDryRun) { - String tblName = new String("SD_PARAMS"); - + private void printPropURIUpdateSummary(ObjectStore.UpdatePropURIRetVal retVal, String + tablePropKey, boolean isDryRun, String tblName, String methodName) { if (retVal == null) { - System.err.println("Encountered error while executing updateMStorageDescriptorTblPropURI - " + - "commit of JDO transaction failed. Failed to update FSRoot locations in " + - "value field corresponding to" + tablePropKey + " in " + tblName + " table."); + System.err.println("Encountered error while executing " + methodName + " - " + + "commit of JDO transaction failed. Failed to update FSRoot locations in " + + "value field corresponding to" + tablePropKey + " in " + tblName + " table."); } else { Map updateLocations = retVal.getUpdateLocations(); if (isDryRun) { @@ -278,7 +275,7 @@ private void printTblPropURIUpdateSummary( List badRecords = retVal.getBadRecords(); if (badRecords.size() > 0) { System.err.println("Warning: Found records with bad " + tablePropKey + " key in " + - tblName + " table.. "); + tblName + " table.. "); for (String badRecord:badRecords) { System.err.println("bad location URI: " + badRecord); } @@ -318,8 +315,8 @@ private void printSerdePropURIUpdateSummary(ObjectStore.UpdateSerdeURIRetVal ret } } - public void updateFSRootLocation(URI oldURI, URI newURI, String serdePropKey, - String tablePropKey, boolean isDryRun) { + public void updateFSRootLocation(URI oldURI, URI newURI, String serdePropKey, String + tablePropKey, boolean isDryRun) { HiveConf hiveConf = new HiveConf(HiveMetaTool.class); initObjectStore(hiveConf); @@ -334,12 +331,20 @@ public void updateFSRootLocation(URI oldURI, URI newURI, String serdePropKey, printTblURIUpdateSummary(updateTblURIRetVal, isDryRun); if (tablePropKey != null) { + System.out.println("Looking for value of " + tablePropKey + " key in TABLE_PARAMS table " + + "to update.."); + ObjectStore.UpdatePropURIRetVal updateTblPropURIRetVal = + objStore.updateTblPropURI(oldURI, newURI, + tablePropKey, isDryRun); + printPropURIUpdateSummary(updateTblPropURIRetVal, tablePropKey, isDryRun, "TABLE_PARAMS", + "updateTblPropURI"); + System.out.println("Looking for value of " + tablePropKey + " key in SD_PARAMS table " + "to update.."); - ObjectStore.UpdateMStorageDescriptorTblPropURIRetVal updateTblPropURIRetVal = - objStore.updateMStorageDescriptorTblPropURI(oldURI, newURI, - tablePropKey, isDryRun); - printTblPropURIUpdateSummary(updateTblPropURIRetVal, tablePropKey, isDryRun); + ObjectStore.UpdatePropURIRetVal updatePropURIRetVal = objStore + .updateMStorageDescriptorTblPropURI(oldURI, newURI, tablePropKey, isDryRun); + printPropURIUpdateSummary(updatePropURIRetVal, tablePropKey, isDryRun, "SD_PARAMS", + "updateMStorageDescriptorTblPropURI"); } if (serdePropKey != null) {