diff --git metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 251d4ba..77a48fb 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -4173,16 +4173,20 @@ public class ObjectStore implements RawStore, Configurable { } private int updateAvroSerdeURI(URI oldLoc, URI newLoc, - HashMap updateLocations, boolean dryRun) { + HashMap updateLocations, String key, boolean dryRun) { int count = 0; Query query = pm.newQuery(MSerDeInfo.class); List mSerdes = (List) query.execute(); pm.retrieveAll(mSerdes); - LOG.info("Looking for location in the value field of schema.url key in SERDES table..."); + if (key != null) { + LOG.info("Looking for location in the value field of "+ key + " key in SERDES table..."); + } else { + LOG.info("Looking for location in the value field of schema.url/avro.schema.url key in " + + "SERDES table..."); + } for(MSerDeInfo mSerde:mSerdes) { - String key = new String("schema.url"); String schemaLoc = mSerde.getParameters().get(key); if (schemaLoc != null) { URI schemaLocURI = null; @@ -4220,7 +4224,7 @@ public class ObjectStore implements RawStore, Configurable { */ @SuppressWarnings("finally") public int updateFSRootLocation(URI oldLoc, URI newLoc, - HashMap updateLocations, boolean dryRun) { + HashMap updateLocations, String serdeKey, boolean dryRun) { boolean committed = false; int count = 0; int totalCount = 0; @@ -4240,8 +4244,18 @@ public class ObjectStore implements RawStore, Configurable { count = updateMStorageDescriptorURI(oldLoc, newLoc, updateLocations, dryRun); totalCount += count; - // upgrade schema.url for avro serde - count = updateAvroSerdeURI(oldLoc, newLoc, updateLocations, dryRun); + // upgrade schema.url/avro.schema.url for avro serde + if (serdeKey != null) { + count = updateAvroSerdeURI(oldLoc, newLoc, updateLocations, serdeKey, dryRun); + } else { + // update records with schema.url key + String key = new String("schema.url"); + count = updateAvroSerdeURI(oldLoc, newLoc, updateLocations, key, dryRun ); + + //update records with avro.schema.url key + key = new String("avro.schema.url"); + count += updateAvroSerdeURI(oldLoc, newLoc, updateLocations, key, dryRun ); + } totalCount += count; committed = commitTransaction(); diff --git metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java index a76594a..3728c6d 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java @@ -22,6 +22,7 @@ import java.net.URI; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; +import java.util.Properties; import java.util.Set; import org.apache.commons.cli.CommandLine; @@ -77,22 +78,34 @@ public class HiveMetaTool { .withArgName("new-loc> " + " hdfsRoots = objStore.listFSRoots(); if (hdfsRoots != null) { - System.out.println("HiveMetaTool:Listing FS Roots.."); + System.out.println("Listing FS Roots.."); for (String s : hdfsRoots) { System.out.println(s); } } else { - System.err.println("HiveMetaTool:Encountered error during listFSRoot - " + + System.err.println("Encountered error during listFSRoot - " + "commit of JDO transaction failed"); } } @@ -125,45 +138,53 @@ public class HiveMetaTool { System.out.println(o.toString()); } } else { - System.err.println("HiveMetaTool:Encountered error during executeJDOQLSelect -" + + System.err.println("Encountered error during executeJDOQLSelect -" + "commit of JDO transaction failed."); } } - private void executeJDOQLUpdate(String query) { + private long executeJDOQLUpdate(String query) { long numUpdated = objStore.executeJDOQLUpdate(query); if (numUpdated >= 0) { - System.out.println("HiveMetaTool:Number of records updated: " + numUpdated); + System.out.println("Number of records updated: " + numUpdated); } else { - System.err.println("HiveMetaTool:Encountered error during executeJDOQL -" + + System.err.println("Encountered error during executeJDOQL -" + "commit of JDO transaction failed."); } + return numUpdated; } private void printUpdateLocations(HashMap updateLocations) { for (String key: updateLocations.keySet()) { String value = updateLocations.get(key); - System.out.println("current location: " + key + " new location: " + value); + System.out.println("Current location: " + key + " New location: " + value); } } - private void updateFSRootLocation(URI oldURI, URI newURI, boolean dryRun) { + private int updateFSRootLocation(URI oldURI, URI newURI, String serdeKey, boolean dryRun) { HashMap updateLocations = new HashMap(); - int count = objStore.updateFSRootLocation(oldURI, newURI, updateLocations, dryRun); + int count = objStore.updateFSRootLocation(oldURI, newURI, updateLocations, serdeKey, dryRun); if (count == -1) { - System.err.println("HiveMetaTool:Encountered error while executing updateFSRootLocation - " + - "commit of JDO transaction failed, failed to update FS Root locations."); + System.err.println("Encountered error while executing updateFSRootLocation - " + + "commit of JDO transaction failed, failed to update FSRoot locations."); } else { if (!dryRun) { - System.out.println("HiveMetaTool: Successfully updated " + count + "FS Root locations"); + System.out.println("Successfully updated " + count + "FSRoot locations"); } else { + System.out.println("Found a total of " + count + " FSRoot locations to update"); printUpdateLocations(updateLocations); } } + return count; } - public static void main(String[] args) { + private static void printAndExit(HiveMetaTool metaTool) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("metatool", metaTool.cmdLineOptions); + System.exit(1); + } + public static void main(String[] args) { HiveConf hiveConf = new HiveConf(HiveMetaTool.class); HiveMetaTool metaTool = new HiveMetaTool(); metaTool.init(); @@ -175,19 +196,31 @@ public class HiveMetaTool { line = parser.parse(metaTool.cmdLineOptions, args); } catch (ParseException e) { System.err.println("HiveMetaTool:Parsing failed. Reason: " + e.getLocalizedMessage()); - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("metatool", metaTool.cmdLineOptions); - System.exit(1); + printAndExit(metaTool); } if (line.hasOption("help")) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("metatool", metaTool.cmdLineOptions); } else if (line.hasOption("listFSRoot")) { + if (line.hasOption("dryRun")) { + System.err.println("HiveMetaTool: dryRun is not valid with listFSRoot"); + printAndExit(metaTool); + } else if (line.hasOption("serdeParamKey")) { + System.err.println("HiveMetaTool: serdeParamKey is not valid with listFSRoot"); + printAndExit(metaTool); + } metaTool.initObjectStore(hiveConf); metaTool.listFSRoot(); } else if (line.hasOption("executeJDOQL")) { String query = line.getOptionValue("executeJDOQL"); + if (line.hasOption("dryRun")) { + System.err.println("HiveMetaTool: dryRun is not valid with executeJDOQL"); + printAndExit(metaTool); + } else if (line.hasOption("serdeParamKey")) { + System.err.println("HiveMetaTool: serdeParamKey is not valid with executeJDOQL"); + printAndExit(metaTool); + } metaTool.initObjectStore(hiveConf); if (query.toLowerCase().trim().startsWith("select")) { metaTool.executeJDOQLSelect(query); @@ -195,15 +228,18 @@ public class HiveMetaTool { metaTool.executeJDOQLUpdate(query); } else { System.err.println("HiveMetaTool:Unsupported statement type"); + printAndExit(metaTool); } } else if (line.hasOption("updateLocation")) { String[] loc = line.getOptionValues("updateLocation"); - boolean dryRun = false; + boolean isDryRun = false; + String serdeKey = null; if (loc.length != 2 && loc.length != 3) { System.err.println("HiveMetaTool:updateLocation takes in 2 required and 1 " + "optional arguements but " + "was passed " + loc.length + " arguements"); + printAndExit(metaTool); } Path newPath = new Path(loc[0]); @@ -213,7 +249,19 @@ public class HiveMetaTool { URI newURI = newPath.toUri(); if (line.hasOption("dryRun")) { - dryRun = true; + isDryRun = true; + } + + if (line.hasOption("serdeParamKey")) { + Properties serdeProps = line.getOptionProperties("serdeParamKey"); + for (String propKey : serdeProps.stringPropertyNames()) { + if (propKey.equals("serde_param_key")) { + serdeKey = serdeProps.getProperty(propKey); + } else { + System.err.println("HiveMetaTool: Invalid property: " + propKey); + printAndExit(metaTool); + } + } } /* @@ -230,19 +278,24 @@ public class HiveMetaTool { System.err.println("HiveMetaTool:A valid scheme is required in both old-loc and new-loc"); } else { metaTool.initObjectStore(hiveConf); - metaTool.updateFSRootLocation(oldURI, newURI, dryRun); + metaTool.updateFSRootLocation(oldURI, newURI, serdeKey, isDryRun); } } else { - System.err.print("HiveMetaTool:Invalid option:" + line.getOptions()); - for (String s : line.getArgs()) { - System.err.print(s + " "); + if (line.hasOption("dryRun")) { + System.err.println("HiveMetaTool: dryRun is not a valid standalone option"); + } else if (line.hasOption("serdeParamKey")) { + System.err.println("HiveMetaTool: serdeParamKey is not a valid standalone option"); + } else { + System.err.print("HiveMetaTool:Parsing failed. Reason: Invalid arguments: " ); + for (String s : line.getArgs()) { + System.err.print(s + " "); + } + System.err.println(); } - System.err.println(); - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("metatool", metaTool.cmdLineOptions); + printAndExit(metaTool); } } finally { metaTool.shutdownObjectStore(); } } -} +} \ No newline at end of file diff --git metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java index 5790ae9..aabdf08 100644 --- metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java +++ metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java @@ -82,6 +82,7 @@ public class TestHiveMetaTool extends TestCase { dropDatabase(dbName); client.createDatabase(db); locationUri = db.getLocationUri(); + String avroUri = locationUri + "/ab.svc"; client.dropType(typeName); Type typ1 = new Type(); @@ -110,10 +111,11 @@ public class TestHiveMetaTool extends TestCase { sd.getSerdeInfo().setParameters(new HashMap()); sd.getSerdeInfo().getParameters().put( org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "1"); + sd.getSerdeInfo().getParameters().put( + org.apache.hadoop.hive.serde.Constants.AVRO_SCHEMA_URL, avroUri); sd.getSerdeInfo().setSerializationLib( - org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName()); + org.apache.hadoop.hive.serde2.avro.AvroSerDe.class.getName()); tbl.setPartitionKeys(new ArrayList()); - client.createTable(tbl); client.close(); } catch (Exception e) { diff --git serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/Constants.java serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/Constants.java index 2c32999..95ea3c0 100644 --- serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/Constants.java +++ serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/Constants.java @@ -5,21 +5,8 @@ */ package org.apache.hadoop.hive.serde; -import org.apache.commons.lang.builder.HashCodeBuilder; -import java.util.List; -import java.util.ArrayList; -import java.util.Map; -import java.util.HashMap; -import java.util.EnumMap; -import java.util.Set; import java.util.HashSet; -import java.util.EnumSet; -import java.util.Collections; -import java.util.BitSet; -import java.nio.ByteBuffer; -import java.util.Arrays; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import java.util.Set; public class Constants { @@ -89,6 +76,8 @@ public class Constants { public static final String LIST_COLUMN_TYPES = "columns.types"; + public static final String AVRO_SCHEMA_URL = "avro.schema.url"; + public static final Set PrimitiveTypes = new HashSet(); static { PrimitiveTypes.add("void");