diff --git bin/ext/metatool.sh bin/ext/metatool.sh new file mode 100755 index 0000000..64011bc --- /dev/null +++ bin/ext/metatool.sh @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +THISSERVICE=metatool +export SERVICE_LIST="${SERVICE_LIST}${THISSERVICE} " + +metatool () { + + CLASS=org.apache.hadoop.hive.metastore.tools.HiveMetaTool + execHiveCmd $CLASS "$@" +} + +metatool_help () { + CLASS=org.apache.hadoop.hive.metastore.tools.HiveMetaTool + execHiveCmd $CLASS "--help" +} diff --git bin/metatool bin/metatool new file mode 100755 index 0000000..df85300 --- /dev/null +++ bin/metatool @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +hive --service metatool "$@" diff --git build.xml build.xml index 6712af9..1d378bc 100644 --- build.xml +++ build.xml @@ -419,6 +419,7 @@ + diff --git eclipse-templates/TestHiveMetaTool.launchtemplate eclipse-templates/TestHiveMetaTool.launchtemplate new file mode 100644 index 0000000..f6429af --- /dev/null +++ eclipse-templates/TestHiveMetaTool.launchtemplate @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git metastore/ivy.xml metastore/ivy.xml index 3011d2f..1e42550 100644 --- metastore/ivy.xml +++ metastore/ivy.xml @@ -45,7 +45,7 @@ - diff --git metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 045b550..251d4ba 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -20,6 +20,8 @@ package org.apache.hadoop.hive.metastore; import static org.apache.commons.lang.StringUtils.join; +import java.net.URI; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -3974,6 +3976,285 @@ public class ObjectStore implements RawStore, Configurable { return join(storedVals,','); } + /** The following API + * + * - executeJDOQLSelect + * + * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift. + * + */ + @SuppressWarnings("finally") + public Collection executeJDOQLSelect(String query) { + boolean committed = false; + Collection result = null; + + LOG.info("Executing query: " + query); + + try { + openTransaction(); + Query q = pm.newQuery(query); + result = (Collection) q.execute(); + committed = commitTransaction(); + } finally { + if (!committed) { + rollbackTransaction(); + return null; + } else { + return result; + } + } + } + + /** The following API + * + * - executeJDOQLUpdate + * + * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift. + * + */ + @SuppressWarnings("finally") + public long executeJDOQLUpdate(String query) { + boolean committed = false; + long numUpdated = 0; + + LOG.info("Executing query: " + query); + + try { + openTransaction(); + Query q = pm.newQuery(query); + numUpdated = (Long) q.execute(); + committed = commitTransaction(); + } finally { + if (!committed) { + rollbackTransaction(); + return -1; + } else { + return numUpdated; + } + } + } + + /** The following API + * + * - listFSRoots + * + * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift. + * + */ + @SuppressWarnings("finally") + public Set listFSRoots() { + boolean committed = false; + Set fsRoots = new HashSet(); + + try { + openTransaction(); + Query query = pm.newQuery(MDatabase.class); + List mDBs = (List) query.execute(); + pm.retrieveAll(mDBs); + + for (MDatabase mDB:mDBs) { + fsRoots.add(mDB.getLocationUri()); + } + committed = commitTransaction(); + } finally { + if (!committed) { + rollbackTransaction(); + return null; + } else { + return fsRoots; + } + } + } + + private boolean shouldUpdateURI(URI onDiskUri, URI inputUri) { + String onDiskHost = onDiskUri.getHost(); + String inputHost = inputUri.getHost(); + + int onDiskPort = onDiskUri.getPort(); + int inputPort = inputUri.getPort(); + + String onDiskScheme = onDiskUri.getScheme(); + String inputScheme = inputUri.getScheme(); + + //compare ports + if (inputPort != -1) { + if (inputPort != onDiskPort) { + return false; + } + } + //compare schemes + if (inputScheme != null) { + if (onDiskScheme == null) { + return false; + } + if (!inputScheme.equalsIgnoreCase(onDiskScheme)) { + return false; + } + } + //compare hosts + if (onDiskHost != null) { + if (!inputHost.equalsIgnoreCase(onDiskHost)) { + return false; + } + } else { + return false; + } + return true; + } + + private int updateMDatabaseURI(URI oldLoc, URI newLoc, + HashMap updateLocations, boolean dryRun) { + int count = 0; + Query query = pm.newQuery(MDatabase.class); + List mDBs = (List) query.execute(); + pm.retrieveAll(mDBs); + + LOG.info("Looking for location in DB_LOCATION_URI field in DBS table..."); + + for(MDatabase mDB:mDBs) { + URI locationURI = null; + try { + locationURI = new URI(mDB.getLocationUri()); + } catch(URISyntaxException e) { + LOG.error("Encountered error while validating location URI" + + e.getLocalizedMessage()); + } + // locationURI is a valid URI + if (locationURI != null) { + if (shouldUpdateURI(locationURI, oldLoc)) { + String dbLoc = mDB.getLocationUri().replaceAll(oldLoc.toString(), newLoc.toString()); + if (dryRun) { + updateLocations.put(locationURI.toString(), dbLoc); + } else { + mDB.setLocationUri(dbLoc); + } + count++; + } + } + } + + LOG.info("Found " + count + " records to update"); + return count; + } + + private int updateMStorageDescriptorURI(URI oldLoc, URI newLoc, + HashMap updateLocations, boolean dryRun) { + int count = 0; + Query query = pm.newQuery(MStorageDescriptor.class); + List mSDSs = (List) query.execute(); + pm.retrieveAll(mSDSs); + + LOG.info("Looking for location in LOCATION field in SDS table..."); + + for(MStorageDescriptor mSDS:mSDSs) { + URI locationURI = null; + try { + locationURI = new URI(mSDS.getLocation()); + } catch (URISyntaxException e) { + LOG.error("Encountered error while validating location URI" + + e.getLocalizedMessage()); + } + // locationURI is a valid URI + if (locationURI != null) { + if (shouldUpdateURI(locationURI, oldLoc)) { + String tblLoc = mSDS.getLocation().replaceAll(oldLoc.toString(), newLoc.toString()); + if (dryRun) { + updateLocations.put(locationURI.toString(), tblLoc); + } else { + mSDS.setLocation(tblLoc); + } + count++; + } + } + } + + LOG.info("Found " + count + " records to update"); + return count; + } + + private int updateAvroSerdeURI(URI oldLoc, URI newLoc, + HashMap updateLocations, boolean dryRun) { + int count = 0; + Query query = pm.newQuery(MSerDeInfo.class); + List mSerdes = (List) query.execute(); + pm.retrieveAll(mSerdes); + + LOG.info("Looking for location in the value field of schema.url key in SERDES table..."); + + for(MSerDeInfo mSerde:mSerdes) { + String key = new String("schema.url"); + String schemaLoc = mSerde.getParameters().get(key); + if (schemaLoc != null) { + URI schemaLocURI = null; + try { + schemaLocURI = new URI(schemaLoc); + } catch (URISyntaxException e) { + LOG.error("Encountered error while validating location URI" + + e.getLocalizedMessage()); + } + // schemaLocURI is a valid URI + if (schemaLocURI != null) { + if (shouldUpdateURI(schemaLocURI, oldLoc)) { + String newSchemaLoc = schemaLoc.replaceAll(oldLoc.toString(), newLoc.toString()); + if (dryRun) { + updateLocations.put(schemaLocURI.toString(), newSchemaLoc); + } else { + mSerde.getParameters().put(key, newSchemaLoc); + } + count++; + } + } + } + } + + LOG.info("Found " + count + " records to update"); + return count; + } + + /** The following APIs + * + * - updateFSRootLocation + * + * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift. + * + */ + @SuppressWarnings("finally") + public int updateFSRootLocation(URI oldLoc, URI newLoc, + HashMap updateLocations, boolean dryRun) { + boolean committed = false; + int count = 0; + int totalCount = 0; + + LOG.info("Old FS root location: " + oldLoc.toString() + + " New FS root location: " + newLoc.toString()); + LOG.info("Updating FS root location..."); + + try { + openTransaction(); + + // update locationURI in mDatabase + count = updateMDatabaseURI(oldLoc, newLoc, updateLocations, dryRun); + totalCount += count; + + // update location in mStorageDescriptor + count = updateMStorageDescriptorURI(oldLoc, newLoc, updateLocations, dryRun); + totalCount += count; + + // upgrade schema.url for avro serde + count = updateAvroSerdeURI(oldLoc, newLoc, updateLocations, dryRun); + totalCount += count; + + committed = commitTransaction(); + } finally { + if (!committed) { + rollbackTransaction(); + return -1; + } else { + return totalCount; + } + } + } + @Override public long cleanupEvents() { boolean commited = false; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java new file mode 100644 index 0000000..a76594a --- /dev/null +++ metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java @@ -0,0 +1,248 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.tools; + +import java.net.URI; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Set; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.ObjectStore; + +/** + * This class provides Hive admins a tool to + * - execute JDOQL against the metastore using DataNucleus + * - perform HA name node upgrade + */ + +public class HiveMetaTool { + + private static final Log LOG = LogFactory.getLog(HiveMetaTool.class.getName()); + private final Options cmdLineOptions = new Options(); + private ObjectStore objStore; + + + public HiveMetaTool() { + } + + @SuppressWarnings("static-access") + private void init() { + + System.out.println("Initializing HiveMetaTool.."); + + Option help = new Option("help", "print this message"); + Option listFSRoot = new Option("listFSRoot", "print the current FS root locations"); + Option executeJDOQL = + OptionBuilder.withArgName("query-string") + .hasArgs() + .withDescription("execute the given JDOQL query") + .create("executeJDOQL"); + + /* Ideally we want to specify the different arguments to updateLocation as separate argNames. + * However if we did that, HelpFormatter swallows all but the last argument. Note that this is + * a know issue with the HelpFormatter class that has not been fixed. We specify all arguments + * with a single argName to workaround this HelpFormatter bug. + */ + Option updateFSRootLoc = + OptionBuilder + .withArgName("new-loc> " + " hdfsRoots = objStore.listFSRoots(); + if (hdfsRoots != null) { + System.out.println("HiveMetaTool:Listing FS Roots.."); + for (String s : hdfsRoots) { + System.out.println(s); + } + } else { + System.err.println("HiveMetaTool:Encountered error during listFSRoot - " + + "commit of JDO transaction failed"); + } + } + + private void executeJDOQLSelect(String query) { + Collection result = objStore.executeJDOQLSelect(query); + if (result != null) { + Iterator iter = result.iterator(); + while (iter.hasNext()) { + Object o = iter.next(); + System.out.println(o.toString()); + } + } else { + System.err.println("HiveMetaTool:Encountered error during executeJDOQLSelect -" + + "commit of JDO transaction failed."); + } + } + + private void executeJDOQLUpdate(String query) { + long numUpdated = objStore.executeJDOQLUpdate(query); + if (numUpdated >= 0) { + System.out.println("HiveMetaTool:Number of records updated: " + numUpdated); + } else { + System.err.println("HiveMetaTool:Encountered error during executeJDOQL -" + + "commit of JDO transaction failed."); + } + } + + private void printUpdateLocations(HashMap updateLocations) { + for (String key: updateLocations.keySet()) { + String value = updateLocations.get(key); + System.out.println("current location: " + key + " new location: " + value); + } + } + + private void updateFSRootLocation(URI oldURI, URI newURI, boolean dryRun) { + HashMap updateLocations = new HashMap(); + int count = objStore.updateFSRootLocation(oldURI, newURI, updateLocations, dryRun); + if (count == -1) { + System.err.println("HiveMetaTool:Encountered error while executing updateFSRootLocation - " + + "commit of JDO transaction failed, failed to update FS Root locations."); + } else { + if (!dryRun) { + System.out.println("HiveMetaTool: Successfully updated " + count + "FS Root locations"); + } else { + printUpdateLocations(updateLocations); + } + } + } + + public static void main(String[] args) { + + HiveConf hiveConf = new HiveConf(HiveMetaTool.class); + HiveMetaTool metaTool = new HiveMetaTool(); + metaTool.init(); + CommandLineParser parser = new GnuParser(); + CommandLine line = null; + + try { + try { + line = parser.parse(metaTool.cmdLineOptions, args); + } catch (ParseException e) { + System.err.println("HiveMetaTool:Parsing failed. Reason: " + e.getLocalizedMessage()); + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("metatool", metaTool.cmdLineOptions); + System.exit(1); + } + + if (line.hasOption("help")) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("metatool", metaTool.cmdLineOptions); + } else if (line.hasOption("listFSRoot")) { + metaTool.initObjectStore(hiveConf); + metaTool.listFSRoot(); + } else if (line.hasOption("executeJDOQL")) { + String query = line.getOptionValue("executeJDOQL"); + metaTool.initObjectStore(hiveConf); + if (query.toLowerCase().trim().startsWith("select")) { + metaTool.executeJDOQLSelect(query); + } else if (query.toLowerCase().trim().startsWith("update")) { + metaTool.executeJDOQLUpdate(query); + } else { + System.err.println("HiveMetaTool:Unsupported statement type"); + } + } else if (line.hasOption("updateLocation")) { + String[] loc = line.getOptionValues("updateLocation"); + boolean dryRun = false; + + if (loc.length != 2 && loc.length != 3) { + System.err.println("HiveMetaTool:updateLocation takes in 2 required and 1 " + + "optional arguements but " + + "was passed " + loc.length + " arguements"); + } + + Path newPath = new Path(loc[0]); + Path oldPath = new Path(loc[1]); + + URI oldURI = oldPath.toUri(); + URI newURI = newPath.toUri(); + + if (line.hasOption("dryRun")) { + dryRun = true; + } + + /* + * validate input - if the old uri contains a valid port, the new uri + * should contain a valid port as well. Both new and old uri should + * contain valid host names and valid schemes. + */ + if (oldURI.getHost() == null || newURI.getHost() == null) { + System.err.println("HiveMetaTool:A valid host is required in both old-loc and new-loc"); + } else if (oldURI.getPort() > 0 && newURI.getPort() < 0) { + System.err.println("HiveMetaTool:old-loc has a valid port, new-loc should " + + "also contain a valid port"); + } else if (oldURI.getScheme() == null || newURI.getScheme() == null) { + System.err.println("HiveMetaTool:A valid scheme is required in both old-loc and new-loc"); + } else { + metaTool.initObjectStore(hiveConf); + metaTool.updateFSRootLocation(oldURI, newURI, dryRun); + } + } else { + System.err.print("HiveMetaTool:Invalid option:" + line.getOptions()); + for (String s : line.getArgs()) { + System.err.print(s + " "); + } + System.err.println(); + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("metatool", metaTool.cmdLineOptions); + } + } finally { + metaTool.shutdownObjectStore(); + } + } +} diff --git metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index 369eebe..925938d 100644 --- metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -1573,161 +1573,163 @@ public abstract class TestHiveMetaStore extends TestCase { * Tests for list partition by filter functionality. * @throws Exception */ - public void testPartitionFilter() throws Exception { - String dbName = "filterdb"; - String tblName = "filtertbl"; - List vals = new ArrayList(3); - vals.add("p11"); - vals.add("p21"); - vals.add("p31"); - List vals2 = new ArrayList(3); - vals2.add("p11"); - vals2.add("p22"); - vals2.add("p31"); - List vals3 = new ArrayList(3); - vals3.add("p12"); - vals3.add("p21"); - vals3.add("p31"); - List vals4 = new ArrayList(3); - vals4.add("p12"); - vals4.add("p23"); - vals4.add("p31"); - List vals5 = new ArrayList(3); - vals5.add("p13"); - vals5.add("p24"); - vals5.add("p31"); - List vals6 = new ArrayList(3); - vals6.add("p13"); - vals6.add("p25"); - vals6.add("p31"); - - silentDropDatabase(dbName); - - Database db = new Database(); - db.setName(dbName); - client.createDatabase(db); - - ArrayList cols = new ArrayList(2); - cols.add(new FieldSchema("c1", Constants.STRING_TYPE_NAME, "")); - cols.add(new FieldSchema("c2", Constants.INT_TYPE_NAME, "")); + public void testPartitionFilter() throws Exception { + String dbName = "filterdb"; + String tblName = "filtertbl"; + + List vals = new ArrayList(3); + vals.add("p11"); + vals.add("p21"); + vals.add("p31"); + List vals2 = new ArrayList(3); + vals2.add("p11"); + vals2.add("p22"); + vals2.add("p31"); + List vals3 = new ArrayList(3); + vals3.add("p12"); + vals3.add("p21"); + vals3.add("p31"); + List vals4 = new ArrayList(3); + vals4.add("p12"); + vals4.add("p23"); + vals4.add("p31"); + List vals5 = new ArrayList(3); + vals5.add("p13"); + vals5.add("p24"); + vals5.add("p31"); + List vals6 = new ArrayList(3); + vals6.add("p13"); + vals6.add("p25"); + vals6.add("p31"); + + silentDropDatabase(dbName); - ArrayList partCols = new ArrayList(3); - partCols.add(new FieldSchema("p1", Constants.STRING_TYPE_NAME, "")); - partCols.add(new FieldSchema("p2", Constants.STRING_TYPE_NAME, "")); - partCols.add(new FieldSchema("p3", Constants.INT_TYPE_NAME, "")); + Database db = new Database(); + db.setName(dbName); + client.createDatabase(db); - Table tbl = new Table(); - tbl.setDbName(dbName); - tbl.setTableName(tblName); - StorageDescriptor sd = new StorageDescriptor(); - tbl.setSd(sd); - sd.setCols(cols); - sd.setCompressed(false); - sd.setNumBuckets(1); - sd.setParameters(new HashMap()); - sd.setBucketCols(new ArrayList()); - sd.setSerdeInfo(new SerDeInfo()); - sd.getSerdeInfo().setName(tbl.getTableName()); - sd.getSerdeInfo().setParameters(new HashMap()); - sd.getSerdeInfo().getParameters() - .put(Constants.SERIALIZATION_FORMAT, "1"); - sd.setSortCols(new ArrayList()); + ArrayList cols = new ArrayList(2); + cols.add(new FieldSchema("c1", Constants.STRING_TYPE_NAME, "")); + cols.add(new FieldSchema("c2", Constants.INT_TYPE_NAME, "")); - tbl.setPartitionKeys(partCols); - client.createTable(tbl); + ArrayList partCols = new ArrayList(3); + partCols.add(new FieldSchema("p1", Constants.STRING_TYPE_NAME, "")); + partCols.add(new FieldSchema("p2", Constants.STRING_TYPE_NAME, "")); + partCols.add(new FieldSchema("p3", Constants.INT_TYPE_NAME, "")); - tbl = client.getTable(dbName, tblName); + Table tbl = new Table(); + tbl.setDbName(dbName); + tbl.setTableName(tblName); + StorageDescriptor sd = new StorageDescriptor(); + tbl.setSd(sd); + sd.setCols(cols); + sd.setCompressed(false); + sd.setNumBuckets(1); + sd.setParameters(new HashMap()); + sd.setBucketCols(new ArrayList()); + sd.setSerdeInfo(new SerDeInfo()); + sd.getSerdeInfo().setName(tbl.getTableName()); + sd.getSerdeInfo().setParameters(new HashMap()); + sd.getSerdeInfo().getParameters() + .put(Constants.SERIALIZATION_FORMAT, "1"); + sd.setSortCols(new ArrayList()); - add_partition(client, tbl, vals, "part1"); - add_partition(client, tbl, vals2, "part2"); - add_partition(client, tbl, vals3, "part3"); - add_partition(client, tbl, vals4, "part4"); - add_partition(client, tbl, vals5, "part5"); - add_partition(client, tbl, vals6, "part6"); + tbl.setPartitionKeys(partCols); + client.createTable(tbl); - checkFilter(client, dbName, tblName, "p1 = \"p11\"", 2); - checkFilter(client, dbName, tblName, "p1 = \"p12\"", 2); - checkFilter(client, dbName, tblName, "p2 = \"p21\"", 2); - checkFilter(client, dbName, tblName, "p2 = \"p23\"", 1); - checkFilter(client, dbName, tblName, "p1 = \"p11\" and p2=\"p22\"", 1); - checkFilter(client, dbName, tblName, "p1 = \"p11\" or p2=\"p23\"", 3); - checkFilter(client, dbName, tblName, "p1 = \"p11\" or p1=\"p12\"", 4); - - checkFilter(client, dbName, tblName, - "p1 = \"p11\" or (p1=\"p12\" and p2=\"p21\")", 3); - checkFilter(client, dbName, tblName, - "p1 = \"p11\" or (p1=\"p12\" and p2=\"p21\") Or " + - "(p1=\"p13\" aNd p2=\"p24\")", 4); - //test for and or precedence - checkFilter(client, dbName, tblName, - "p1=\"p12\" and (p2=\"p27\" Or p2=\"p21\")", 1); - checkFilter(client, dbName, tblName, - "p1=\"p12\" and p2=\"p27\" Or p2=\"p21\"", 2); - - checkFilter(client, dbName, tblName, "p1 > \"p12\"", 2); - checkFilter(client, dbName, tblName, "p1 >= \"p12\"", 4); - checkFilter(client, dbName, tblName, "p1 < \"p12\"", 2); - checkFilter(client, dbName, tblName, "p1 <= \"p12\"", 4); - checkFilter(client, dbName, tblName, "p1 <> \"p12\"", 4); - checkFilter(client, dbName, tblName, "p1 like \"p1.*\"", 6); - checkFilter(client, dbName, tblName, "p2 like \"p.*3\"", 1); - - //Test for setting the maximum partition count - List partitions = client.listPartitionsByFilter(dbName, - tblName, "p1 >= \"p12\"", (short) 2); - assertEquals("User specified row limit for partitions", - 2, partitions.size()); + tbl = client.getTable(dbName, tblName); + + add_partition(client, tbl, vals, "part1"); + add_partition(client, tbl, vals2, "part2"); + add_partition(client, tbl, vals3, "part3"); + add_partition(client, tbl, vals4, "part4"); + add_partition(client, tbl, vals5, "part5"); + add_partition(client, tbl, vals6, "part6"); + + checkFilter(client, dbName, tblName, "p1 = \"p11\"", 2); + checkFilter(client, dbName, tblName, "p1 = \"p12\"", 2); + checkFilter(client, dbName, tblName, "p2 = \"p21\"", 2); + checkFilter(client, dbName, tblName, "p2 = \"p23\"", 1); + checkFilter(client, dbName, tblName, "p1 = \"p11\" and p2=\"p22\"", 1); + checkFilter(client, dbName, tblName, "p1 = \"p11\" or p2=\"p23\"", 3); + checkFilter(client, dbName, tblName, "p1 = \"p11\" or p1=\"p12\"", 4); + + checkFilter(client, dbName, tblName, + "p1 = \"p11\" or (p1=\"p12\" and p2=\"p21\")", 3); + checkFilter(client, dbName, tblName, + "p1 = \"p11\" or (p1=\"p12\" and p2=\"p21\") Or " + + "(p1=\"p13\" aNd p2=\"p24\")", 4); + //test for and or precedence + checkFilter(client, dbName, tblName, + "p1=\"p12\" and (p2=\"p27\" Or p2=\"p21\")", 1); + checkFilter(client, dbName, tblName, + "p1=\"p12\" and p2=\"p27\" Or p2=\"p21\"", 2); + + checkFilter(client, dbName, tblName, "p1 > \"p12\"", 2); + checkFilter(client, dbName, tblName, "p1 >= \"p12\"", 4); + checkFilter(client, dbName, tblName, "p1 < \"p12\"", 2); + checkFilter(client, dbName, tblName, "p1 <= \"p12\"", 4); + checkFilter(client, dbName, tblName, "p1 <> \"p12\"", 4); + checkFilter(client, dbName, tblName, "p1 like \"p1.*\"", 6); + checkFilter(client, dbName, tblName, "p2 like \"p.*3\"", 1); + + //Test for setting the maximum partition count + List partitions = client.listPartitionsByFilter(dbName, + tblName, "p1 >= \"p12\"", (short) 2); + assertEquals("User specified row limit for partitions", + 2, partitions.size()); - //Negative tests - Exception me = null; - try { - client.listPartitionsByFilter(dbName, - tblName, "p3 >= \"p12\"", (short) -1); - } catch(MetaException e) { - me = e; - } - assertNotNull(me); - assertTrue("Filter on int partition key", me.getMessage().contains( - "Filtering is supported only on partition keys of type string")); + //Negative tests + Exception me = null; + try { + client.listPartitionsByFilter(dbName, + tblName, "p3 >= \"p12\"", (short) -1); + } catch(MetaException e) { + me = e; + } + assertNotNull(me); + assertTrue("Filter on int partition key", me.getMessage().contains( + "Filtering is supported only on partition keys of type string")); - me = null; - try { - client.listPartitionsByFilter(dbName, - tblName, "c1 >= \"p12\"", (short) -1); - } catch(MetaException e) { - me = e; - } - assertNotNull(me); - assertTrue("Filter on invalid key", me.getMessage().contains( - " is not a partitioning key for the table")); + me = null; + try { + client.listPartitionsByFilter(dbName, + tblName, "c1 >= \"p12\"", (short) -1); + } catch(MetaException e) { + me = e; + } + assertNotNull(me); + assertTrue("Filter on invalid key", me.getMessage().contains( + " is not a partitioning key for the table")); - me = null; - try { - client.listPartitionsByFilter(dbName, - tblName, "c1 >= ", (short) -1); - } catch(MetaException e) { - me = e; - } - assertNotNull(me); - assertTrue("Invalid filter string", me.getMessage().contains( - "Error parsing partition filter")); + me = null; + try { + client.listPartitionsByFilter(dbName, + tblName, "c1 >= ", (short) -1); + } catch(MetaException e) { + me = e; + } + assertNotNull(me); + assertTrue("Invalid filter string", me.getMessage().contains( + "Error parsing partition filter")); - me = null; - try { - client.listPartitionsByFilter("invDBName", - "invTableName", "p1 = \"p11\"", (short) -1); - } catch(NoSuchObjectException e) { - me = e; - } - assertNotNull(me); - assertTrue("NoSuchObject exception", me.getMessage().contains( - "database/table does not exist")); + me = null; + try { + client.listPartitionsByFilter("invDBName", + "invTableName", "p1 = \"p11\"", (short) -1); + } catch(NoSuchObjectException e) { + me = e; + } + assertNotNull(me); + assertTrue("NoSuchObject exception", me.getMessage().contains( + "database/table does not exist")); - client.dropTable(dbName, tblName); - client.dropDatabase(dbName); + client.dropTable(dbName, tblName); + client.dropDatabase(dbName); } + /** * Test filtering on table with single partition * @throws Exception @@ -2299,5 +2301,4 @@ public abstract class TestHiveMetaStore extends TestCase { createPartitions(dbName, tbl, values); } - } diff --git metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java new file mode 100644 index 0000000..5790ae9 --- /dev/null +++ metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java @@ -0,0 +1,224 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.HashMap; + +import junit.framework.TestCase; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.InvalidOperationException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.SerDeInfo; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.Type; +import org.apache.hadoop.hive.metastore.tools.HiveMetaTool; +import org.apache.hadoop.hive.serde.Constants; +import org.apache.hadoop.util.StringUtils; + +public class TestHiveMetaTool extends TestCase { + + private HiveMetaStoreClient client; + + private PrintStream originalOut; + private OutputStream os; + private PrintStream ps; + private String locationUri; + + + private void dropDatabase(String dbName) throws Exception { + try { + client.dropDatabase(dbName); + } catch (NoSuchObjectException e) { + } catch (InvalidOperationException e) { + } catch (Exception e) { + throw e; + } + } + + @Override + protected void setUp() throws Exception { + super.setUp(); + + try { + HiveConf hiveConf = new HiveConf(HiveMetaTool.class); + client = new HiveMetaStoreClient(hiveConf, null); + + // Setup output stream to redirect output to + os = new ByteArrayOutputStream(); + ps = new PrintStream(os); + + // create a dummy database and a couple of dummy tables + String dbName = "testDB"; + String typeName = "Person"; + String tblName = "simpleTbl"; + + Database db = new Database(); + db.setName(dbName); + client.dropTable(dbName, tblName); + dropDatabase(dbName); + client.createDatabase(db); + locationUri = db.getLocationUri(); + + client.dropType(typeName); + Type typ1 = new Type(); + typ1.setName(typeName); + typ1.setFields(new ArrayList(2)); + typ1.getFields().add( + new FieldSchema("name", Constants.STRING_TYPE_NAME, "")); + typ1.getFields().add( + new FieldSchema("income", Constants.INT_TYPE_NAME, "")); + client.createType(typ1); + + Table tbl = new Table(); + tbl.setDbName(dbName); + tbl.setTableName(tblName); + StorageDescriptor sd = new StorageDescriptor(); + tbl.setSd(sd); + sd.setCols(typ1.getFields()); + sd.setCompressed(false); + sd.setNumBuckets(1); + sd.setParameters(new HashMap()); + sd.getParameters().put("test_param_1", "Use this for comments etc"); + sd.setBucketCols(new ArrayList(2)); + sd.getBucketCols().add("name"); + sd.setSerdeInfo(new SerDeInfo()); + sd.getSerdeInfo().setName(tbl.getTableName()); + sd.getSerdeInfo().setParameters(new HashMap()); + sd.getSerdeInfo().getParameters().put( + org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "1"); + sd.getSerdeInfo().setSerializationLib( + org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName()); + tbl.setPartitionKeys(new ArrayList()); + + client.createTable(tbl); + client.close(); + } catch (Exception e) { + System.err.println("Unable to setup the hive metatool test"); + System.err.println(StringUtils.stringifyException(e)); + throw new Exception(e); + } + } + + private void redirectOutputStream() { + + originalOut = System.out; + System.setOut(ps); + + } + + private void restoreOutputStream() { + + System.setOut(originalOut); + } + + public void testListFSRoot() throws Exception { + + redirectOutputStream(); + String[] args = new String[1]; + args[0] = new String("-listFSRoot"); + + try { + HiveMetaTool.main(args); + String out = os.toString(); + boolean b = out.contains(locationUri); + assertTrue(b); + } finally { + restoreOutputStream(); + System.out.println("Completed testListFSRoot"); + } + } + + public void testExecuteJDOQL() throws Exception { + + redirectOutputStream(); + String[] args = new String[2]; + args[0] = new String("-executeJDOQL"); + args[1] = new String("select locationUri from org.apache.hadoop.hive.metastore.model.MDatabase"); + + try { + HiveMetaTool.main(args); + String out = os.toString(); + boolean b = out.contains(locationUri); + assertTrue(b); + } finally { + restoreOutputStream(); + System.out.println("Completed testExecuteJDOQL"); + } + } + + public void testUpdateFSRootLocation() throws Exception { + + redirectOutputStream(); + String newLocationUri = "hdfs://nn-ha-uri/user/hive/warehouse"; + String[] args = new String[3]; + args[0] = new String("-updateLocation"); + args[1] = new String(newLocationUri); + args[2] = new String(locationUri); + + String[] args2 = new String[1]; + args2[0] = new String("-listFSRoot"); + + try { + + // perform HA upgrade + HiveMetaTool.main(args); + + // obtain new HDFS root + HiveMetaTool.main(args2); + + String out = os.toString(); + boolean b = out.contains(newLocationUri); + + if (b) { + System.out.println("updateFSRootLocation successful"); + } else { + System.out.println("updateFSRootLocation failed"); + } + // restore the original HDFS root if needed + if (b) { + args[1] = new String(locationUri); + args[2] = new String(newLocationUri); + HiveMetaTool.main(args); + } + } finally { + restoreOutputStream(); + System.out.println("Completed testUpdateFSRootLocation.."); + } + } + + @Override + protected void tearDown() throws Exception { + try { + super.tearDown(); + + } catch (Throwable e) { + System.err.println("Unable to close metastore"); + System.err.println(StringUtils.stringifyException(e)); + throw new Exception(e); + } + } +}