diff --git metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 251d4ba..fe6500d 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -3983,24 +3983,23 @@ public class ObjectStore implements RawStore, Configurable { * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift. * */ - @SuppressWarnings("finally") public Collection executeJDOQLSelect(String query) { boolean committed = false; Collection result = null; - LOG.info("Executing query: " + query); - try { openTransaction(); Query q = pm.newQuery(query); result = (Collection) q.execute(); committed = commitTransaction(); + if (committed) { + return result; + } else { + return null; + } } finally { if (!committed) { rollbackTransaction(); - return null; - } else { - return result; } } } @@ -4012,24 +4011,23 @@ public class ObjectStore implements RawStore, Configurable { * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift. * */ - @SuppressWarnings("finally") public long executeJDOQLUpdate(String query) { boolean committed = false; long numUpdated = 0; - LOG.info("Executing query: " + query); - try { openTransaction(); Query q = pm.newQuery(query); numUpdated = (Long) q.execute(); committed = commitTransaction(); + if (committed) { + return numUpdated; + } else { + return -1; + } } finally { if (!committed) { rollbackTransaction(); - return -1; - } else { - return numUpdated; } } } @@ -4041,7 +4039,6 @@ public class ObjectStore implements RawStore, Configurable { * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift. * */ - @SuppressWarnings("finally") public Set listFSRoots() { boolean committed = false; Set fsRoots = new HashSet(); @@ -4056,12 +4053,14 @@ public class ObjectStore implements RawStore, Configurable { fsRoots.add(mDB.getLocationUri()); } committed = commitTransaction(); + if (committed) { + return fsRoots; + } else { + return null; + } } finally { if (!committed) { rollbackTransaction(); - return null; - } else { - return fsRoots; } } } @@ -4102,155 +4101,322 @@ public class ObjectStore implements RawStore, Configurable { return true; } - private int updateMDatabaseURI(URI oldLoc, URI newLoc, - HashMap updateLocations, boolean dryRun) { - int count = 0; - Query query = pm.newQuery(MDatabase.class); - List mDBs = (List) query.execute(); - pm.retrieveAll(mDBs); + public class UpdateMDatabaseURIRetVal { + private List badRecords; + private Map updateLocations; - LOG.info("Looking for location in DB_LOCATION_URI field in DBS table..."); + UpdateMDatabaseURIRetVal(List badRecords, Map updateLocations) { + this.badRecords = badRecords; + this.updateLocations = updateLocations; + } - for(MDatabase mDB:mDBs) { - URI locationURI = null; - try { - locationURI = new URI(mDB.getLocationUri()); - } catch(URISyntaxException e) { - LOG.error("Encountered error while validating location URI" - + e.getLocalizedMessage()); - } - // locationURI is a valid URI - if (locationURI != null) { - if (shouldUpdateURI(locationURI, oldLoc)) { - String dbLoc = mDB.getLocationUri().replaceAll(oldLoc.toString(), newLoc.toString()); - if (dryRun) { + public List getBadRecords() { + return badRecords; + } + + public void setBadRecords(List badRecords) { + this.badRecords = badRecords; + } + + public Map getUpdateLocations() { + return updateLocations; + } + + public void setUpdateLocations(Map updateLocations) { + this.updateLocations = updateLocations; + } + } + + /** The following APIs + * + * - updateMDatabaseURI + * + * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift. + * + */ + public UpdateMDatabaseURIRetVal updateMDatabaseURI(URI oldLoc, URI newLoc, boolean dryRun) { + boolean committed = false; + Map updateLocations = new HashMap(); + List badRecords = new ArrayList(); + UpdateMDatabaseURIRetVal retVal = null; + + try { + openTransaction(); + Query query = pm.newQuery(MDatabase.class); + List mDBs = (List) query.execute(); + pm.retrieveAll(mDBs); + + for(MDatabase mDB:mDBs) { + URI locationURI = null; + String location = mDB.getLocationUri(); + try { + locationURI = new URI(location); + } catch(URISyntaxException e) { + badRecords.add(location); + } + if (locationURI == null) { + badRecords.add(location); + } else { + if (shouldUpdateURI(locationURI, oldLoc)) { + String dbLoc = mDB.getLocationUri().replaceAll(oldLoc.toString(), newLoc.toString()); updateLocations.put(locationURI.toString(), dbLoc); - } else { - mDB.setLocationUri(dbLoc); + if (!dryRun) { + mDB.setLocationUri(dbLoc); + } } - count++; } } + committed = commitTransaction(); + if (committed) { + retVal = new UpdateMDatabaseURIRetVal(badRecords, updateLocations); + } + return retVal; + } finally { + if (!committed) { + rollbackTransaction(); + } } - - LOG.info("Found " + count + " records to update"); - return count; } - private int updateMStorageDescriptorURI(URI oldLoc, URI newLoc, - HashMap updateLocations, boolean dryRun) { - int count = 0; - Query query = pm.newQuery(MStorageDescriptor.class); - List mSDSs = (List) query.execute(); - pm.retrieveAll(mSDSs); + public class UpdateMStorageDescriptorTblPropURIRetVal { + private List badRecords; + private Map updateLocations; - LOG.info("Looking for location in LOCATION field in SDS table..."); + UpdateMStorageDescriptorTblPropURIRetVal(List badRecords, + Map updateLocations) { + this.badRecords = badRecords; + this.updateLocations = updateLocations; + } - for(MStorageDescriptor mSDS:mSDSs) { - URI locationURI = null; - try { - locationURI = new URI(mSDS.getLocation()); - } catch (URISyntaxException e) { - LOG.error("Encountered error while validating location URI" - + e.getLocalizedMessage()); - } - // locationURI is a valid URI - if (locationURI != null) { - if (shouldUpdateURI(locationURI, oldLoc)) { - String tblLoc = mSDS.getLocation().replaceAll(oldLoc.toString(), newLoc.toString()); - if (dryRun) { - updateLocations.put(locationURI.toString(), tblLoc); - } else { - mSDS.setLocation(tblLoc); + public List getBadRecords() { + return badRecords; + } + + public void setBadRecords(List badRecords) { + this.badRecords = badRecords; + } + + public Map getUpdateLocations() { + return updateLocations; + } + + public void setUpdateLocations(Map updateLocations) { + this.updateLocations = updateLocations; + } + } + + /** The following APIs + * + * - updateMStorageDescriptorTblPropURI + * + * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift. + * + */ + public UpdateMStorageDescriptorTblPropURIRetVal updateMStorageDescriptorTblPropURI(URI oldLoc, + URI newLoc, String tblPropKey, boolean isDryRun) { + boolean committed = false; + Map updateLocations = new HashMap(); + List badRecords = new ArrayList(); + UpdateMStorageDescriptorTblPropURIRetVal retVal = null; + + try { + openTransaction(); + Query query = pm.newQuery(MStorageDescriptor.class); + List mSDSs = (List) query.execute(); + pm.retrieveAll(mSDSs); + + for(MStorageDescriptor mSDS:mSDSs) { + URI tablePropLocationURI = null; + String tablePropLocation = mSDS.getParameters().get(tblPropKey); + if (tablePropLocation != null) { + try { + tablePropLocationURI = new URI(tablePropLocation); + } catch (URISyntaxException e) { + badRecords.add(tablePropLocation); + } } - count++; + // if tablePropKey that was passed in lead to a valid URI resolution, update it if + //parts of it match the old-NN-loc, else add to badRecords + if (tablePropLocationURI == null) { + badRecords.add(tablePropLocation); + } else { + if (shouldUpdateURI(tablePropLocationURI, oldLoc)) { + String tblPropLoc = mSDS.getParameters().get(tblPropKey).replaceAll(oldLoc.toString(), + newLoc.toString()); + updateLocations.put(tablePropLocationURI.toString(), tblPropLoc); + if (!isDryRun) { + mSDS.getParameters().put(tblPropKey, tblPropLoc); + } + } + } + } + committed = commitTransaction(); + if (committed) { + retVal = new UpdateMStorageDescriptorTblPropURIRetVal(badRecords, updateLocations); + } + return retVal; + } finally { + if (!committed) { + rollbackTransaction(); } - } + } + } + + public class UpdateMStorageDescriptorTblURIRetVal { + private List badRecords; + private Map updateLocations; + + UpdateMStorageDescriptorTblURIRetVal(List badRecords, + Map updateLocations) { + this.badRecords = badRecords; + this.updateLocations = updateLocations; } - LOG.info("Found " + count + " records to update"); - return count; + public List getBadRecords() { + return badRecords; + } + + public void setBadRecords(List badRecords) { + this.badRecords = badRecords; + } + + public Map getUpdateLocations() { + return updateLocations; + } + + public void setUpdateLocations(Map updateLocations) { + this.updateLocations = updateLocations; + } } - private int updateAvroSerdeURI(URI oldLoc, URI newLoc, - HashMap updateLocations, boolean dryRun) { - int count = 0; - Query query = pm.newQuery(MSerDeInfo.class); - List mSerdes = (List) query.execute(); - pm.retrieveAll(mSerdes); + /** The following APIs + * + * - updateMStorageDescriptorTblURI + * + * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift. + * + */ + public UpdateMStorageDescriptorTblURIRetVal updateMStorageDescriptorTblURI(URI oldLoc, URI newLoc, + boolean isDryRun) { + boolean committed = false; + Map updateLocations = new HashMap(); + List badRecords = new ArrayList(); + UpdateMStorageDescriptorTblURIRetVal retVal = null; - LOG.info("Looking for location in the value field of schema.url key in SERDES table..."); + try { + openTransaction(); + Query query = pm.newQuery(MStorageDescriptor.class); + List mSDSs = (List) query.execute(); + pm.retrieveAll(mSDSs); - for(MSerDeInfo mSerde:mSerdes) { - String key = new String("schema.url"); - String schemaLoc = mSerde.getParameters().get(key); - if (schemaLoc != null) { - URI schemaLocURI = null; + for(MStorageDescriptor mSDS:mSDSs) { + URI locationURI = null; + String location = mSDS.getLocation(); try { - schemaLocURI = new URI(schemaLoc); + locationURI = new URI(location); } catch (URISyntaxException e) { - LOG.error("Encountered error while validating location URI" - + e.getLocalizedMessage()); + badRecords.add(location); } - // schemaLocURI is a valid URI - if (schemaLocURI != null) { - if (shouldUpdateURI(schemaLocURI, oldLoc)) { - String newSchemaLoc = schemaLoc.replaceAll(oldLoc.toString(), newLoc.toString()); - if (dryRun) { - updateLocations.put(schemaLocURI.toString(), newSchemaLoc); - } else { - mSerde.getParameters().put(key, newSchemaLoc); + if (locationURI == null) { + badRecords.add(location); + } else { + if (shouldUpdateURI(locationURI, oldLoc)) { + String tblLoc = mSDS.getLocation().replaceAll(oldLoc.toString(), newLoc.toString()); + updateLocations.put(locationURI.toString(), tblLoc); + if (!isDryRun) { + mSDS.setLocation(tblLoc); } - count++; } } } + committed = commitTransaction(); + if (committed) { + retVal = new UpdateMStorageDescriptorTblURIRetVal(badRecords, updateLocations); + } + return retVal; + } finally { + if (!committed) { + rollbackTransaction(); + } + } + } + + public class UpdateSerdeURIRetVal { + private List badRecords; + private Map updateLocations; + + UpdateSerdeURIRetVal(List badRecords, Map updateLocations) { + this.badRecords = badRecords; + this.updateLocations = updateLocations; + } + + public List getBadRecords() { + return badRecords; + } + + public void setBadRecords(List badRecords) { + this.badRecords = badRecords; + } + + public Map getUpdateLocations() { + return updateLocations; } - LOG.info("Found " + count + " records to update"); - return count; + public void setUpdateLocations(Map updateLocations) { + this.updateLocations = updateLocations; + } } /** The following APIs * - * - updateFSRootLocation + * - updateSerdeURI * * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift. * */ - @SuppressWarnings("finally") - public int updateFSRootLocation(URI oldLoc, URI newLoc, - HashMap updateLocations, boolean dryRun) { + public UpdateSerdeURIRetVal updateSerdeURI(URI oldLoc, URI newLoc, String serdeProp, + boolean isDryRun) { boolean committed = false; - int count = 0; - int totalCount = 0; - - LOG.info("Old FS root location: " + oldLoc.toString() + - " New FS root location: " + newLoc.toString()); - LOG.info("Updating FS root location..."); + Map updateLocations = new HashMap(); + List badRecords = new ArrayList(); + UpdateSerdeURIRetVal retVal = null; try { openTransaction(); - - // update locationURI in mDatabase - count = updateMDatabaseURI(oldLoc, newLoc, updateLocations, dryRun); - totalCount += count; - - // update location in mStorageDescriptor - count = updateMStorageDescriptorURI(oldLoc, newLoc, updateLocations, dryRun); - totalCount += count; - - // upgrade schema.url for avro serde - count = updateAvroSerdeURI(oldLoc, newLoc, updateLocations, dryRun); - totalCount += count; - + Query query = pm.newQuery(MSerDeInfo.class); + List mSerdes = (List) query.execute(); + pm.retrieveAll(mSerdes); + + for(MSerDeInfo mSerde:mSerdes) { + String schemaLoc = mSerde.getParameters().get(serdeProp); + if (schemaLoc != null) { + URI schemaLocURI = null; + try { + schemaLocURI = new URI(schemaLoc); + } catch (URISyntaxException e) { + badRecords.add(schemaLoc); + } + if (schemaLocURI == null) { + badRecords.add(schemaLoc); + } else { + if (shouldUpdateURI(schemaLocURI, oldLoc)) { + String newSchemaLoc = schemaLoc.replaceAll(oldLoc.toString(), newLoc.toString()); + updateLocations.put(schemaLocURI.toString(), newSchemaLoc); + if (!isDryRun) { + mSerde.getParameters().put(serdeProp, newSchemaLoc); + } + } + } + } + } committed = commitTransaction(); + if (committed) { + retVal = new UpdateSerdeURIRetVal(badRecords, updateLocations); + } + return retVal; } finally { if (!committed) { rollbackTransaction(); - return -1; - } else { - return totalCount; } } } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java index a76594a..a0500f1 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java @@ -20,8 +20,9 @@ package org.apache.hadoop.hive.metastore.tools; import java.net.URI; import java.util.Collection; -import java.util.HashMap; import java.util.Iterator; +import java.util.List; +import java.util.Map; import java.util.Set; import org.apache.commons.cli.CommandLine; @@ -49,9 +50,10 @@ public class HiveMetaTool { private static final Log LOG = LogFactory.getLog(HiveMetaTool.class.getName()); private final Options cmdLineOptions = new Options(); private ObjectStore objStore; - + private boolean isObjStoreInitialized; public HiveMetaTool() { + this.isObjStoreInitialized = false; } @SuppressWarnings("static-access") @@ -75,48 +77,78 @@ public class HiveMetaTool { Option updateFSRootLoc = OptionBuilder .withArgName("new-loc> " + " hdfsRoots = objStore.listFSRoots(); if (hdfsRoots != null) { - System.out.println("HiveMetaTool:Listing FS Roots.."); + System.out.println("Listing FS Roots.."); for (String s : hdfsRoots) { System.out.println(s); } } else { - System.err.println("HiveMetaTool:Encountered error during listFSRoot - " + + System.err.println("Encountered error during listFSRoot - " + "commit of JDO transaction failed"); } } private void executeJDOQLSelect(String query) { + HiveConf hiveConf = new HiveConf(HiveMetaTool.class); + initObjectStore(hiveConf); + + System.out.println("Executing query: " + query); Collection result = objStore.executeJDOQLSelect(query); if (result != null) { Iterator iter = result.iterator(); @@ -125,46 +157,205 @@ public class HiveMetaTool { System.out.println(o.toString()); } } else { - System.err.println("HiveMetaTool:Encountered error during executeJDOQLSelect -" + + System.err.println("Encountered error during executeJDOQLSelect -" + "commit of JDO transaction failed."); } } - private void executeJDOQLUpdate(String query) { + private long executeJDOQLUpdate(String query) { + HiveConf hiveConf = new HiveConf(HiveMetaTool.class); + initObjectStore(hiveConf); + + System.out.println("Executing query: " + query); long numUpdated = objStore.executeJDOQLUpdate(query); if (numUpdated >= 0) { - System.out.println("HiveMetaTool:Number of records updated: " + numUpdated); + System.out.println("Number of records updated: " + numUpdated); } else { - System.err.println("HiveMetaTool:Encountered error during executeJDOQL -" + + System.err.println("Encountered error during executeJDOQL -" + "commit of JDO transaction failed."); } + return numUpdated; } - private void printUpdateLocations(HashMap updateLocations) { + private int printUpdateLocations(Map updateLocations) { + int count = 0; for (String key: updateLocations.keySet()) { String value = updateLocations.get(key); - System.out.println("current location: " + key + " new location: " + value); + System.out.println("old location: " + key + " new location: " + value); + count++; } + return count; } - private void updateFSRootLocation(URI oldURI, URI newURI, boolean dryRun) { - HashMap updateLocations = new HashMap(); - int count = objStore.updateFSRootLocation(oldURI, newURI, updateLocations, dryRun); - if (count == -1) { - System.err.println("HiveMetaTool:Encountered error while executing updateFSRootLocation - " + - "commit of JDO transaction failed, failed to update FS Root locations."); + private void printTblURIUpdateSummary(ObjectStore.UpdateMStorageDescriptorTblURIRetVal retVal, + boolean isDryRun) { + String tblName = new String("SDS"); + String fieldName = new String("LOCATION"); + + if (retVal == null) { + System.err.println("Encountered error while executing updateMStorageDescriptorTblURI - " + + "commit of JDO transaction failed. Failed to update FSRoot locations in " + + fieldName + "field in " + tblName + " table."); } else { - if (!dryRun) { - System.out.println("HiveMetaTool: Successfully updated " + count + "FS Root locations"); + Map updateLocations = retVal.getUpdateLocations(); + if (isDryRun) { + System.out.println("Dry Run of updateLocation on table " + tblName + ".."); + } else { + System.out.println("Successfully updated the following locations.."); + } + int count = printUpdateLocations(updateLocations); + if (isDryRun) { + System.out.println("Found " + count + " records in " + tblName + " table to update"); } else { - printUpdateLocations(updateLocations); + System.out.println("Updated " + count + " records in " + tblName + " table"); + } + List badRecords = retVal.getBadRecords(); + if (badRecords.size() > 0) { + System.err.println("Found records with bad " + fieldName + " in " + tblName + " table.. "); + for (String badRecord:badRecords) { + System.err.println("bad location URI: " + badRecord); + } } } } - public static void main(String[] args) { + private void printDatabaseURIUpdateSummary(ObjectStore.UpdateMDatabaseURIRetVal retVal, + boolean isDryRun) { + String tblName = new String("DBS"); + String fieldName = new String("LOCATION_URI"); + if (retVal == null) { + System.err.println("Encountered error while executing updateMDatabaseURI - " + + "commit of JDO transaction failed. Failed to update FSRoot locations in " + + fieldName + "field in " + tblName + " table."); + } else { + Map updateLocations = retVal.getUpdateLocations(); + if (isDryRun) { + System.out.println("Dry Run of updateLocation on table " + tblName + ".."); + } else { + System.out.println("Successfully updated the following locations.."); + } + int count = printUpdateLocations(updateLocations); + if (isDryRun) { + System.out.println("Found " + count + " records in " + tblName + " table to update"); + } else { + System.out.println("Updated " + count + " records in " + tblName + " table"); + } + List badRecords = retVal.getBadRecords(); + if (badRecords.size() > 0) { + System.err.println("Found records with bad " + fieldName + " in " + tblName + " table.. "); + for (String badRecord:badRecords) { + System.err.println("bad location URI: " + badRecord); + } + } + } + } + + private void printTblPropURIUpdateSummary( + ObjectStore.UpdateMStorageDescriptorTblPropURIRetVal retVal, String tablePropKey, + boolean isDryRun) { + String tblName = new String("SD_PARAMS"); + + if (retVal == null) { + System.err.println("Encountered error while executing updateMStorageDescriptorTblPropURI - " + + "commit of JDO transaction failed. Failed to update FSRoot locations in " + + "value field corresponding to" + tablePropKey + " in " + tblName + " table."); + } else { + Map updateLocations = retVal.getUpdateLocations(); + if (isDryRun) { + System.out.println("Dry Run of updateLocation on table " + tblName + ".."); + } else { + System.out.println("Successfully updated the following locations.."); + } + int count = printUpdateLocations(updateLocations); + if (isDryRun) { + System.out.println("Found " + count + " records in " + tblName + " table to update"); + } else { + System.out.println("Updated " + count + " records in " + tblName + " table"); + } + List badRecords = retVal.getBadRecords(); + if (badRecords.size() > 0) { + System.err.println("Found records with bad " + tablePropKey + " key in " + + tblName + " table.. "); + for (String badRecord:badRecords) { + System.err.println("bad location URI: " + badRecord); + } + } + } + } + + private void printSerdePropURIUpdateSummary(ObjectStore.UpdateSerdeURIRetVal retVal, + String serdePropKey, boolean isDryRun) { + String tblName = new String("SERDE_PARAMS"); + + if (retVal == null) { + System.err.println("Encountered error while executing updateSerdeURI - " + + "commit of JDO transaction failed. Failed to update FSRoot locations in " + + "value field corresponding to " + serdePropKey + " in " + tblName + " table."); + } else { + Map updateLocations = retVal.getUpdateLocations(); + if (isDryRun) { + System.out.println("Dry Run of updateLocation on table " + tblName + ".."); + } else { + System.out.println("Successfully updated the following locations.."); + } + int count = printUpdateLocations(updateLocations); + if (isDryRun) { + System.out.println("Found " + count + " records in " + tblName + " table to update"); + } else { + System.out.println("Updated " + count + " records in " + tblName + " table"); + } + List badRecords = retVal.getBadRecords(); + if (badRecords.size() > 0) { + System.err.println("Found records with bad " + serdePropKey + " key in " + + tblName + " table.. "); + for (String badRecord:badRecords) { + System.err.println("bad location URI: " + badRecord); + } + } + } + } + + public void updateFSRootLocation(URI oldURI, URI newURI, String serdePropKey, + String tablePropKey, boolean isDryRun) { HiveConf hiveConf = new HiveConf(HiveMetaTool.class); + initObjectStore(hiveConf); + + System.out.println("Looking for LOCATION_URI field in DBS table to update.."); + ObjectStore.UpdateMDatabaseURIRetVal updateMDBURIRetVal = objStore.updateMDatabaseURI(oldURI, + newURI, isDryRun); + printDatabaseURIUpdateSummary(updateMDBURIRetVal, isDryRun); + + System.out.println("Looking for LOCATION field in SDS table to update.."); + ObjectStore.UpdateMStorageDescriptorTblURIRetVal updateTblURIRetVal = + objStore.updateMStorageDescriptorTblURI(oldURI, newURI, isDryRun); + printTblURIUpdateSummary(updateTblURIRetVal, isDryRun); + + if (tablePropKey != null) { + System.out.println("Looking for value of " + tablePropKey + " key in SD_PARAMS table " + + "to update.."); + ObjectStore.UpdateMStorageDescriptorTblPropURIRetVal updateTblPropURIRetVal = + objStore.updateMStorageDescriptorTblPropURI(oldURI, newURI, + tablePropKey, isDryRun); + printTblPropURIUpdateSummary(updateTblPropURIRetVal, tablePropKey, isDryRun); + } + + if (serdePropKey != null) { + System.out.println("Looking for value of " + serdePropKey + " key in SERDE_PARAMS table " + + "to update.."); + ObjectStore.UpdateSerdeURIRetVal updateSerdeURIretVal = objStore.updateSerdeURI(oldURI, + newURI, serdePropKey, isDryRun); + printSerdePropURIUpdateSummary(updateSerdeURIretVal, serdePropKey, isDryRun); + } + } + + private static void printAndExit(HiveMetaTool metaTool) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("metatool", metaTool.cmdLineOptions); + System.exit(1); + } + + public static void main(String[] args) { HiveMetaTool metaTool = new HiveMetaTool(); metaTool.init(); CommandLineParser parser = new GnuParser(); @@ -175,35 +366,55 @@ public class HiveMetaTool { line = parser.parse(metaTool.cmdLineOptions, args); } catch (ParseException e) { System.err.println("HiveMetaTool:Parsing failed. Reason: " + e.getLocalizedMessage()); - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("metatool", metaTool.cmdLineOptions); - System.exit(1); + printAndExit(metaTool); } if (line.hasOption("help")) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("metatool", metaTool.cmdLineOptions); } else if (line.hasOption("listFSRoot")) { - metaTool.initObjectStore(hiveConf); + if (line.hasOption("dryRun")) { + System.err.println("HiveMetaTool: dryRun is not valid with listFSRoot"); + printAndExit(metaTool); + } else if (line.hasOption("serdePropKey")) { + System.err.println("HiveMetaTool: serdePropKey is not valid with listFSRoot"); + printAndExit(metaTool); + } else if (line.hasOption("tablePropKey")) { + System.err.println("HiveMetaTool: tablePropKey is not valid with listFSRoot"); + printAndExit(metaTool); + } metaTool.listFSRoot(); } else if (line.hasOption("executeJDOQL")) { String query = line.getOptionValue("executeJDOQL"); - metaTool.initObjectStore(hiveConf); + if (line.hasOption("dryRun")) { + System.err.println("HiveMetaTool: dryRun is not valid with executeJDOQL"); + printAndExit(metaTool); + } else if (line.hasOption("serdePropKey")) { + System.err.println("HiveMetaTool: serdePropKey is not valid with executeJDOQL"); + printAndExit(metaTool); + } else if (line.hasOption("tablePropKey")) { + System.err.println("HiveMetaTool: tablePropKey is not valid with executeJDOQL"); + printAndExit(metaTool); + } if (query.toLowerCase().trim().startsWith("select")) { metaTool.executeJDOQLSelect(query); } else if (query.toLowerCase().trim().startsWith("update")) { metaTool.executeJDOQLUpdate(query); } else { System.err.println("HiveMetaTool:Unsupported statement type"); + printAndExit(metaTool); } } else if (line.hasOption("updateLocation")) { String[] loc = line.getOptionValues("updateLocation"); - boolean dryRun = false; + boolean isDryRun = false; + String serdepropKey = null; + String tablePropKey = null; if (loc.length != 2 && loc.length != 3) { System.err.println("HiveMetaTool:updateLocation takes in 2 required and 1 " + - "optional arguements but " + - "was passed " + loc.length + " arguements"); + "optional arguments but " + + "was passed " + loc.length + " arguments"); + printAndExit(metaTool); } Path newPath = new Path(loc[0]); @@ -213,7 +424,15 @@ public class HiveMetaTool { URI newURI = newPath.toUri(); if (line.hasOption("dryRun")) { - dryRun = true; + isDryRun = true; + } + + if (line.hasOption("serdePropKey")) { + serdepropKey = line.getOptionValue("serdePropKey"); + } + + if (line.hasOption("tablePropKey")) { + tablePropKey = line.getOptionValue("tablePropKey"); } /* @@ -229,20 +448,27 @@ public class HiveMetaTool { } else if (oldURI.getScheme() == null || newURI.getScheme() == null) { System.err.println("HiveMetaTool:A valid scheme is required in both old-loc and new-loc"); } else { - metaTool.initObjectStore(hiveConf); - metaTool.updateFSRootLocation(oldURI, newURI, dryRun); + metaTool.updateFSRootLocation(oldURI, newURI, serdepropKey, tablePropKey, isDryRun); } } else { - System.err.print("HiveMetaTool:Invalid option:" + line.getOptions()); - for (String s : line.getArgs()) { - System.err.print(s + " "); + if (line.hasOption("dryRun")) { + System.err.println("HiveMetaTool: dryRun is not a valid standalone option"); + } else if (line.hasOption("serdePropKey")) { + System.err.println("HiveMetaTool: serdePropKey is not a valid standalone option"); + } else if (line.hasOption("tablePropKey")) { + System.err.println("HiveMetaTool: tablePropKey is not a valid standalone option"); + printAndExit(metaTool); + } else { + System.err.print("HiveMetaTool:Parsing failed. Reason: Invalid arguments: " ); + for (String s : line.getArgs()) { + System.err.print(s + " "); + } + System.err.println(); } - System.err.println(); - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("metatool", metaTool.cmdLineOptions); + printAndExit(metaTool); } } finally { metaTool.shutdownObjectStore(); } } -} +} \ No newline at end of file diff --git metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java index 5790ae9..9ef36e8 100644 --- metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java +++ metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.Type; import org.apache.hadoop.hive.metastore.tools.HiveMetaTool; import org.apache.hadoop.hive.serde.Constants; +import org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils; import org.apache.hadoop.util.StringUtils; public class TestHiveMetaTool extends TestCase { @@ -72,16 +73,20 @@ public class TestHiveMetaTool extends TestCase { ps = new PrintStream(os); // create a dummy database and a couple of dummy tables - String dbName = "testDB"; + String dbName = "TestHiveMetaToolDB"; String typeName = "Person"; String tblName = "simpleTbl"; + String badTblName = "badSimpleTbl"; Database db = new Database(); db.setName(dbName); client.dropTable(dbName, tblName); + client.dropTable(dbName, badTblName); dropDatabase(dbName); client.createDatabase(db); locationUri = db.getLocationUri(); + String avroUri = "hdfs://nn.example.com/warehouse/hive/ab.avsc"; + String badAvroUri = new String("hdfs:/hive"); client.dropType(typeName); Type typ1 = new Type(); @@ -110,10 +115,34 @@ public class TestHiveMetaTool extends TestCase { sd.getSerdeInfo().setParameters(new HashMap()); sd.getSerdeInfo().getParameters().put( org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "1"); + sd.getParameters().put(AvroSerdeUtils.SCHEMA_URL, avroUri); sd.getSerdeInfo().setSerializationLib( - org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName()); + org.apache.hadoop.hive.serde2.avro.AvroSerDe.class.getName()); tbl.setPartitionKeys(new ArrayList()); + client.createTable(tbl); + //create a table with bad avro uri + tbl = new Table(); + tbl.setDbName(dbName); + tbl.setTableName(badTblName); + sd = new StorageDescriptor(); + tbl.setSd(sd); + sd.setCols(typ1.getFields()); + sd.setCompressed(false); + sd.setNumBuckets(1); + sd.setParameters(new HashMap()); + sd.getParameters().put("test_param_1", "Use this for comments etc"); + sd.setBucketCols(new ArrayList(2)); + sd.getBucketCols().add("name"); + sd.setSerdeInfo(new SerDeInfo()); + sd.getSerdeInfo().setName(tbl.getTableName()); + sd.getSerdeInfo().setParameters(new HashMap()); + sd.getSerdeInfo().getParameters().put( + org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "1"); + sd.getParameters().put(AvroSerdeUtils.SCHEMA_URL, badAvroUri); + sd.getSerdeInfo().setSerializationLib( + org.apache.hadoop.hive.serde2.avro.AvroSerDe.class.getName()); + tbl.setPartitionKeys(new ArrayList()); client.createTable(tbl); client.close(); } catch (Exception e) { @@ -173,37 +202,29 @@ public class TestHiveMetaTool extends TestCase { public void testUpdateFSRootLocation() throws Exception { redirectOutputStream(); - String newLocationUri = "hdfs://nn-ha-uri/user/hive/warehouse"; - String[] args = new String[3]; + String oldLocationUri = "hdfs://nn.example.com/"; + String newLocationUri = "hdfs://nn-ha-uri/"; + String[] args = new String[5]; args[0] = new String("-updateLocation"); args[1] = new String(newLocationUri); - args[2] = new String(locationUri); - - String[] args2 = new String[1]; - args2[0] = new String("-listFSRoot"); + args[2] = new String(oldLocationUri); + args[3] = new String("-tablePropKey"); + args[4] = new String("avro.schema.url"); try { - // perform HA upgrade HiveMetaTool.main(args); - - // obtain new HDFS root - HiveMetaTool.main(args2); - String out = os.toString(); boolean b = out.contains(newLocationUri); + restoreOutputStream(); + assertTrue(b); - if (b) { - System.out.println("updateFSRootLocation successful"); - } else { - System.out.println("updateFSRootLocation failed"); - } - // restore the original HDFS root if needed - if (b) { - args[1] = new String(locationUri); - args[2] = new String(newLocationUri); - HiveMetaTool.main(args); - } + //restore the original HDFS root + args[1] = new String(oldLocationUri); + args[2] = new String(newLocationUri); + redirectOutputStream(); + HiveMetaTool.main(args); + restoreOutputStream(); } finally { restoreOutputStream(); System.out.println("Completed testUpdateFSRootLocation.."); @@ -214,7 +235,6 @@ public class TestHiveMetaTool extends TestCase { protected void tearDown() throws Exception { try { super.tearDown(); - } catch (Throwable e) { System.err.println("Unable to close metastore"); System.err.println(StringUtils.stringifyException(e));