diff --git metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 251d4ba..6757c1f 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -4138,16 +4138,20 @@ public class ObjectStore implements RawStore, Configurable { } private int updateMStorageDescriptorURI(URI oldLoc, URI newLoc, - HashMap updateLocations, boolean dryRun) { - int count = 0; + HashMap updateLocations, String key, boolean dryRun) { + int count1 = 0; + int count2 = 0; Query query = pm.newQuery(MStorageDescriptor.class); List mSDSs = (List) query.execute(); pm.retrieveAll(mSDSs); LOG.info("Looking for location in LOCATION field in SDS table..."); + LOG.info("Looking for " + key + " key in SD_PARAMS table..."); for(MStorageDescriptor mSDS:mSDSs) { URI locationURI = null; + URI tablePropLocationURI = null; + try { locationURI = new URI(mSDS.getLocation()); } catch (URISyntaxException e) { @@ -4163,26 +4167,51 @@ public class ObjectStore implements RawStore, Configurable { } else { mSDS.setLocation(tblLoc); } - count++; + count1++; + } + } + + if (key != null) { + String tablePropLocation = mSDS.getParameters().get(key); + if (tablePropLocation != null) { + try { + tablePropLocationURI = new URI(tablePropLocation); + } catch (URISyntaxException e) { + LOG.error("Encountered error while validating " + key + " URI" + e.getLocalizedMessage()); + } + } + //tablePropKey that was passed in lead to a valid URI resolution, update it if parts of it + //match the old-NN-loc + if (tablePropLocationURI != null) { + if (shouldUpdateURI(tablePropLocationURI, oldLoc)) { + String tblPropLoc = mSDS.getParameters().get(key).replaceAll(oldLoc.toString(), + newLoc.toString()); + if (dryRun) { + updateLocations.put(tablePropLocationURI.toString(), tblPropLoc); + } else { + mSDS.getParameters().put(key, tblPropLoc); + } + count2++; + } } } } - LOG.info("Found " + count + " records to update"); - return count; + LOG.info("Found " + count1 + " records in SDS table to update"); + LOG.info("Found " + count2 + " records in SD_PARAMS table to update"); + return count1 + count2; } - private int updateAvroSerdeURI(URI oldLoc, URI newLoc, - HashMap updateLocations, boolean dryRun) { + private int updateSerdeURI(URI oldLoc, URI newLoc, + HashMap updateLocations, String key, boolean dryRun) { int count = 0; Query query = pm.newQuery(MSerDeInfo.class); List mSerdes = (List) query.execute(); pm.retrieveAll(mSerdes); - LOG.info("Looking for location in the value field of schema.url key in SERDES table..."); + LOG.info("Looking for location in the value field of "+ key + " key in SERDE_PARAMS table..."); for(MSerDeInfo mSerde:mSerdes) { - String key = new String("schema.url"); String schemaLoc = mSerde.getParameters().get(key); if (schemaLoc != null) { URI schemaLocURI = null; @@ -4207,7 +4236,7 @@ public class ObjectStore implements RawStore, Configurable { } } - LOG.info("Found " + count + " records to update"); + LOG.info("Found " + count + " records in SERDE_PARAMS table to update"); return count; } @@ -4220,7 +4249,7 @@ public class ObjectStore implements RawStore, Configurable { */ @SuppressWarnings("finally") public int updateFSRootLocation(URI oldLoc, URI newLoc, - HashMap updateLocations, boolean dryRun) { + HashMap updateLocations, String serdeKey, String tableProp, boolean dryRun) { boolean committed = false; int count = 0; int totalCount = 0; @@ -4237,12 +4266,14 @@ public class ObjectStore implements RawStore, Configurable { totalCount += count; // update location in mStorageDescriptor - count = updateMStorageDescriptorURI(oldLoc, newLoc, updateLocations, dryRun); + count = updateMStorageDescriptorURI(oldLoc, newLoc, updateLocations, tableProp, dryRun); totalCount += count; - // upgrade schema.url for avro serde - count = updateAvroSerdeURI(oldLoc, newLoc, updateLocations, dryRun); - totalCount += count; + // update serde param key if needed + if (serdeKey != null) { + count = updateSerdeURI(oldLoc, newLoc, updateLocations, serdeKey, dryRun); + totalCount += count; + } committed = commitTransaction(); } finally { diff --git metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java index a76594a..249fa78 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java @@ -75,24 +75,43 @@ public class HiveMetaTool { Option updateFSRootLoc = OptionBuilder .withArgName("new-loc> " + " hdfsRoots = objStore.listFSRoots(); if (hdfsRoots != null) { - System.out.println("HiveMetaTool:Listing FS Roots.."); + System.out.println("Listing FS Roots.."); for (String s : hdfsRoots) { System.out.println(s); } } else { - System.err.println("HiveMetaTool:Encountered error during listFSRoot - " + + System.err.println("Encountered error during listFSRoot - " + "commit of JDO transaction failed"); } } @@ -125,45 +144,55 @@ public class HiveMetaTool { System.out.println(o.toString()); } } else { - System.err.println("HiveMetaTool:Encountered error during executeJDOQLSelect -" + + System.err.println("Encountered error during executeJDOQLSelect -" + "commit of JDO transaction failed."); } } - private void executeJDOQLUpdate(String query) { + private long executeJDOQLUpdate(String query) { long numUpdated = objStore.executeJDOQLUpdate(query); if (numUpdated >= 0) { - System.out.println("HiveMetaTool:Number of records updated: " + numUpdated); + System.out.println("Number of records updated: " + numUpdated); } else { - System.err.println("HiveMetaTool:Encountered error during executeJDOQL -" + + System.err.println("Encountered error during executeJDOQL -" + "commit of JDO transaction failed."); } + return numUpdated; } private void printUpdateLocations(HashMap updateLocations) { for (String key: updateLocations.keySet()) { String value = updateLocations.get(key); - System.out.println("current location: " + key + " new location: " + value); + System.out.println("Current location: " + key + " New location: " + value); } } - private void updateFSRootLocation(URI oldURI, URI newURI, boolean dryRun) { + private int updateFSRootLocation(URI oldURI, URI newURI, String serdeKey, + String tableProp, boolean dryRun) { HashMap updateLocations = new HashMap(); - int count = objStore.updateFSRootLocation(oldURI, newURI, updateLocations, dryRun); + int count = objStore.updateFSRootLocation(oldURI, newURI, updateLocations, + serdeKey, tableProp, dryRun); if (count == -1) { - System.err.println("HiveMetaTool:Encountered error while executing updateFSRootLocation - " + - "commit of JDO transaction failed, failed to update FS Root locations."); + System.err.println("Encountered error while executing updateFSRootLocation - " + + "commit of JDO transaction failed, failed to update FSRoot locations."); } else { if (!dryRun) { - System.out.println("HiveMetaTool: Successfully updated " + count + "FS Root locations"); + System.out.println("Successfully updated " + count + " FSRoot locations"); } else { + System.out.println("Found a total of " + count + " FSRoot locations to update"); printUpdateLocations(updateLocations); } } + return count; } - public static void main(String[] args) { + private static void printAndExit(HiveMetaTool metaTool) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("metatool", metaTool.cmdLineOptions); + System.exit(1); + } + public static void main(String[] args) { HiveConf hiveConf = new HiveConf(HiveMetaTool.class); HiveMetaTool metaTool = new HiveMetaTool(); metaTool.init(); @@ -175,19 +204,37 @@ public class HiveMetaTool { line = parser.parse(metaTool.cmdLineOptions, args); } catch (ParseException e) { System.err.println("HiveMetaTool:Parsing failed. Reason: " + e.getLocalizedMessage()); - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("metatool", metaTool.cmdLineOptions); - System.exit(1); + printAndExit(metaTool); } if (line.hasOption("help")) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("metatool", metaTool.cmdLineOptions); } else if (line.hasOption("listFSRoot")) { + if (line.hasOption("dryRun")) { + System.err.println("HiveMetaTool: dryRun is not valid with listFSRoot"); + printAndExit(metaTool); + } else if (line.hasOption("serdeParamKey")) { + System.err.println("HiveMetaTool: serdeParamKey is not valid with listFSRoot"); + printAndExit(metaTool); + } else if (line.hasOption("tablePropKey")) { + System.err.println("HiveMetaTool: tablePropKey is not valid with listFSRoot"); + printAndExit(metaTool); + } metaTool.initObjectStore(hiveConf); metaTool.listFSRoot(); } else if (line.hasOption("executeJDOQL")) { String query = line.getOptionValue("executeJDOQL"); + if (line.hasOption("dryRun")) { + System.err.println("HiveMetaTool: dryRun is not valid with executeJDOQL"); + printAndExit(metaTool); + } else if (line.hasOption("serdeParamKey")) { + System.err.println("HiveMetaTool: serdeParamKey is not valid with executeJDOQL"); + printAndExit(metaTool); + } else if (line.hasOption("tablePropKey")) { + System.err.println("HiveMetaTool: tablePropKey is not valid with executeJDOQL"); + printAndExit(metaTool); + } metaTool.initObjectStore(hiveConf); if (query.toLowerCase().trim().startsWith("select")) { metaTool.executeJDOQLSelect(query); @@ -195,15 +242,19 @@ public class HiveMetaTool { metaTool.executeJDOQLUpdate(query); } else { System.err.println("HiveMetaTool:Unsupported statement type"); + printAndExit(metaTool); } } else if (line.hasOption("updateLocation")) { String[] loc = line.getOptionValues("updateLocation"); - boolean dryRun = false; + boolean isDryRun = false; + String serdeParamKey = null; + String tablePropKey = null; if (loc.length != 2 && loc.length != 3) { System.err.println("HiveMetaTool:updateLocation takes in 2 required and 1 " + "optional arguements but " + "was passed " + loc.length + " arguements"); + printAndExit(metaTool); } Path newPath = new Path(loc[0]); @@ -213,7 +264,15 @@ public class HiveMetaTool { URI newURI = newPath.toUri(); if (line.hasOption("dryRun")) { - dryRun = true; + isDryRun = true; + } + + if (line.hasOption("serdeParamKey")) { + serdeParamKey = line.getOptionValue("serdeParamKey"); + } + + if (line.hasOption("tablePropKey")) { + tablePropKey = line.getOptionValue("tablePropKey"); } /* @@ -230,19 +289,27 @@ public class HiveMetaTool { System.err.println("HiveMetaTool:A valid scheme is required in both old-loc and new-loc"); } else { metaTool.initObjectStore(hiveConf); - metaTool.updateFSRootLocation(oldURI, newURI, dryRun); + metaTool.updateFSRootLocation(oldURI, newURI, serdeParamKey, tablePropKey, isDryRun); } } else { - System.err.print("HiveMetaTool:Invalid option:" + line.getOptions()); - for (String s : line.getArgs()) { - System.err.print(s + " "); + if (line.hasOption("dryRun")) { + System.err.println("HiveMetaTool: dryRun is not a valid standalone option"); + } else if (line.hasOption("serdeParamKey")) { + System.err.println("HiveMetaTool: serdeParamKey is not a valid standalone option"); + } else if (line.hasOption("tablePropKey")) { + System.err.println("HiveMetaTool: tablePropKey is not a valid standalone option"); + printAndExit(metaTool); + } else { + System.err.print("HiveMetaTool:Parsing failed. Reason: Invalid arguments: " ); + for (String s : line.getArgs()) { + System.err.print(s + " "); + } + System.err.println(); } - System.err.println(); - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("metatool", metaTool.cmdLineOptions); + printAndExit(metaTool); } } finally { metaTool.shutdownObjectStore(); } } -} +} \ No newline at end of file diff --git metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java index 5790ae9..aabdf08 100644 --- metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java +++ metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java @@ -82,6 +82,7 @@ public class TestHiveMetaTool extends TestCase { dropDatabase(dbName); client.createDatabase(db); locationUri = db.getLocationUri(); + String avroUri = locationUri + "/ab.svc"; client.dropType(typeName); Type typ1 = new Type(); @@ -110,10 +111,11 @@ public class TestHiveMetaTool extends TestCase { sd.getSerdeInfo().setParameters(new HashMap()); sd.getSerdeInfo().getParameters().put( org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "1"); + sd.getSerdeInfo().getParameters().put( + org.apache.hadoop.hive.serde.Constants.AVRO_SCHEMA_URL, avroUri); sd.getSerdeInfo().setSerializationLib( - org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName()); + org.apache.hadoop.hive.serde2.avro.AvroSerDe.class.getName()); tbl.setPartitionKeys(new ArrayList()); - client.createTable(tbl); client.close(); } catch (Exception e) {