diff --git bin/ext/metatool.sh bin/ext/metatool.sh
new file mode 100755
index 0000000..64011bc
--- /dev/null
+++ bin/ext/metatool.sh
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+THISSERVICE=metatool
+export SERVICE_LIST="${SERVICE_LIST}${THISSERVICE} "
+
+metatool () {
+
+ CLASS=org.apache.hadoop.hive.metastore.tools.HiveMetaTool
+ execHiveCmd $CLASS "$@"
+}
+
+metatool_help () {
+ CLASS=org.apache.hadoop.hive.metastore.tools.HiveMetaTool
+ execHiveCmd $CLASS "--help"
+}
diff --git bin/metatool bin/metatool
new file mode 100755
index 0000000..df85300
--- /dev/null
+++ bin/metatool
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+hive --service metatool "$@"
diff --git build.xml build.xml
index 6712af9..1d378bc 100644
--- build.xml
+++ build.xml
@@ -419,6 +419,7 @@
+
diff --git eclipse-templates/TestHiveMetaTool.launchtemplate eclipse-templates/TestHiveMetaTool.launchtemplate
new file mode 100644
index 0000000..f6429af
--- /dev/null
+++ eclipse-templates/TestHiveMetaTool.launchtemplate
@@ -0,0 +1,43 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git metastore/ivy.xml metastore/ivy.xml
index 3011d2f..1e42550 100644
--- metastore/ivy.xml
+++ metastore/ivy.xml
@@ -45,7 +45,7 @@
-
diff --git metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 045b550..251d4ba 100644
--- metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.hive.metastore;
import static org.apache.commons.lang.StringUtils.join;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -3974,6 +3976,285 @@ public class ObjectStore implements RawStore, Configurable {
return join(storedVals,',');
}
+ /** The following API
+ *
+ * - executeJDOQLSelect
+ *
+ * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift.
+ *
+ */
+ @SuppressWarnings("finally")
+ public Collection> executeJDOQLSelect(String query) {
+ boolean committed = false;
+ Collection> result = null;
+
+ LOG.info("Executing query: " + query);
+
+ try {
+ openTransaction();
+ Query q = pm.newQuery(query);
+ result = (Collection>) q.execute();
+ committed = commitTransaction();
+ } finally {
+ if (!committed) {
+ rollbackTransaction();
+ return null;
+ } else {
+ return result;
+ }
+ }
+ }
+
+ /** The following API
+ *
+ * - executeJDOQLUpdate
+ *
+ * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift.
+ *
+ */
+ @SuppressWarnings("finally")
+ public long executeJDOQLUpdate(String query) {
+ boolean committed = false;
+ long numUpdated = 0;
+
+ LOG.info("Executing query: " + query);
+
+ try {
+ openTransaction();
+ Query q = pm.newQuery(query);
+ numUpdated = (Long) q.execute();
+ committed = commitTransaction();
+ } finally {
+ if (!committed) {
+ rollbackTransaction();
+ return -1;
+ } else {
+ return numUpdated;
+ }
+ }
+ }
+
+ /** The following API
+ *
+ * - listFSRoots
+ *
+ * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift.
+ *
+ */
+ @SuppressWarnings("finally")
+ public Set listFSRoots() {
+ boolean committed = false;
+ Set fsRoots = new HashSet();
+
+ try {
+ openTransaction();
+ Query query = pm.newQuery(MDatabase.class);
+ List mDBs = (List) query.execute();
+ pm.retrieveAll(mDBs);
+
+ for (MDatabase mDB:mDBs) {
+ fsRoots.add(mDB.getLocationUri());
+ }
+ committed = commitTransaction();
+ } finally {
+ if (!committed) {
+ rollbackTransaction();
+ return null;
+ } else {
+ return fsRoots;
+ }
+ }
+ }
+
+ private boolean shouldUpdateURI(URI onDiskUri, URI inputUri) {
+ String onDiskHost = onDiskUri.getHost();
+ String inputHost = inputUri.getHost();
+
+ int onDiskPort = onDiskUri.getPort();
+ int inputPort = inputUri.getPort();
+
+ String onDiskScheme = onDiskUri.getScheme();
+ String inputScheme = inputUri.getScheme();
+
+ //compare ports
+ if (inputPort != -1) {
+ if (inputPort != onDiskPort) {
+ return false;
+ }
+ }
+ //compare schemes
+ if (inputScheme != null) {
+ if (onDiskScheme == null) {
+ return false;
+ }
+ if (!inputScheme.equalsIgnoreCase(onDiskScheme)) {
+ return false;
+ }
+ }
+ //compare hosts
+ if (onDiskHost != null) {
+ if (!inputHost.equalsIgnoreCase(onDiskHost)) {
+ return false;
+ }
+ } else {
+ return false;
+ }
+ return true;
+ }
+
+ private int updateMDatabaseURI(URI oldLoc, URI newLoc,
+ HashMap updateLocations, boolean dryRun) {
+ int count = 0;
+ Query query = pm.newQuery(MDatabase.class);
+ List mDBs = (List) query.execute();
+ pm.retrieveAll(mDBs);
+
+ LOG.info("Looking for location in DB_LOCATION_URI field in DBS table...");
+
+ for(MDatabase mDB:mDBs) {
+ URI locationURI = null;
+ try {
+ locationURI = new URI(mDB.getLocationUri());
+ } catch(URISyntaxException e) {
+ LOG.error("Encountered error while validating location URI"
+ + e.getLocalizedMessage());
+ }
+ // locationURI is a valid URI
+ if (locationURI != null) {
+ if (shouldUpdateURI(locationURI, oldLoc)) {
+ String dbLoc = mDB.getLocationUri().replaceAll(oldLoc.toString(), newLoc.toString());
+ if (dryRun) {
+ updateLocations.put(locationURI.toString(), dbLoc);
+ } else {
+ mDB.setLocationUri(dbLoc);
+ }
+ count++;
+ }
+ }
+ }
+
+ LOG.info("Found " + count + " records to update");
+ return count;
+ }
+
+ private int updateMStorageDescriptorURI(URI oldLoc, URI newLoc,
+ HashMap updateLocations, boolean dryRun) {
+ int count = 0;
+ Query query = pm.newQuery(MStorageDescriptor.class);
+ List mSDSs = (List) query.execute();
+ pm.retrieveAll(mSDSs);
+
+ LOG.info("Looking for location in LOCATION field in SDS table...");
+
+ for(MStorageDescriptor mSDS:mSDSs) {
+ URI locationURI = null;
+ try {
+ locationURI = new URI(mSDS.getLocation());
+ } catch (URISyntaxException e) {
+ LOG.error("Encountered error while validating location URI"
+ + e.getLocalizedMessage());
+ }
+ // locationURI is a valid URI
+ if (locationURI != null) {
+ if (shouldUpdateURI(locationURI, oldLoc)) {
+ String tblLoc = mSDS.getLocation().replaceAll(oldLoc.toString(), newLoc.toString());
+ if (dryRun) {
+ updateLocations.put(locationURI.toString(), tblLoc);
+ } else {
+ mSDS.setLocation(tblLoc);
+ }
+ count++;
+ }
+ }
+ }
+
+ LOG.info("Found " + count + " records to update");
+ return count;
+ }
+
+ private int updateAvroSerdeURI(URI oldLoc, URI newLoc,
+ HashMap updateLocations, boolean dryRun) {
+ int count = 0;
+ Query query = pm.newQuery(MSerDeInfo.class);
+ List mSerdes = (List) query.execute();
+ pm.retrieveAll(mSerdes);
+
+ LOG.info("Looking for location in the value field of schema.url key in SERDES table...");
+
+ for(MSerDeInfo mSerde:mSerdes) {
+ String key = new String("schema.url");
+ String schemaLoc = mSerde.getParameters().get(key);
+ if (schemaLoc != null) {
+ URI schemaLocURI = null;
+ try {
+ schemaLocURI = new URI(schemaLoc);
+ } catch (URISyntaxException e) {
+ LOG.error("Encountered error while validating location URI"
+ + e.getLocalizedMessage());
+ }
+ // schemaLocURI is a valid URI
+ if (schemaLocURI != null) {
+ if (shouldUpdateURI(schemaLocURI, oldLoc)) {
+ String newSchemaLoc = schemaLoc.replaceAll(oldLoc.toString(), newLoc.toString());
+ if (dryRun) {
+ updateLocations.put(schemaLocURI.toString(), newSchemaLoc);
+ } else {
+ mSerde.getParameters().put(key, newSchemaLoc);
+ }
+ count++;
+ }
+ }
+ }
+ }
+
+ LOG.info("Found " + count + " records to update");
+ return count;
+ }
+
+ /** The following APIs
+ *
+ * - updateFSRootLocation
+ *
+ * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift.
+ *
+ */
+ @SuppressWarnings("finally")
+ public int updateFSRootLocation(URI oldLoc, URI newLoc,
+ HashMap updateLocations, boolean dryRun) {
+ boolean committed = false;
+ int count = 0;
+ int totalCount = 0;
+
+ LOG.info("Old FS root location: " + oldLoc.toString() +
+ " New FS root location: " + newLoc.toString());
+ LOG.info("Updating FS root location...");
+
+ try {
+ openTransaction();
+
+ // update locationURI in mDatabase
+ count = updateMDatabaseURI(oldLoc, newLoc, updateLocations, dryRun);
+ totalCount += count;
+
+ // update location in mStorageDescriptor
+ count = updateMStorageDescriptorURI(oldLoc, newLoc, updateLocations, dryRun);
+ totalCount += count;
+
+ // upgrade schema.url for avro serde
+ count = updateAvroSerdeURI(oldLoc, newLoc, updateLocations, dryRun);
+ totalCount += count;
+
+ committed = commitTransaction();
+ } finally {
+ if (!committed) {
+ rollbackTransaction();
+ return -1;
+ } else {
+ return totalCount;
+ }
+ }
+ }
+
@Override
public long cleanupEvents() {
boolean commited = false;
diff --git metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java
new file mode 100644
index 0000000..a76594a
--- /dev/null
+++ metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java
@@ -0,0 +1,248 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.tools;
+
+import java.net.URI;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Set;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.ObjectStore;
+
+/**
+ * This class provides Hive admins a tool to
+ * - execute JDOQL against the metastore using DataNucleus
+ * - perform HA name node upgrade
+ */
+
+public class HiveMetaTool {
+
+ private static final Log LOG = LogFactory.getLog(HiveMetaTool.class.getName());
+ private final Options cmdLineOptions = new Options();
+ private ObjectStore objStore;
+
+
+ public HiveMetaTool() {
+ }
+
+ @SuppressWarnings("static-access")
+ private void init() {
+
+ System.out.println("Initializing HiveMetaTool..");
+
+ Option help = new Option("help", "print this message");
+ Option listFSRoot = new Option("listFSRoot", "print the current FS root locations");
+ Option executeJDOQL =
+ OptionBuilder.withArgName("query-string")
+ .hasArgs()
+ .withDescription("execute the given JDOQL query")
+ .create("executeJDOQL");
+
+ /* Ideally we want to specify the different arguments to updateLocation as separate argNames.
+ * However if we did that, HelpFormatter swallows all but the last argument. Note that this is
+ * a know issue with the HelpFormatter class that has not been fixed. We specify all arguments
+ * with a single argName to workaround this HelpFormatter bug.
+ */
+ Option updateFSRootLoc =
+ OptionBuilder
+ .withArgName("new-loc> " + " hdfsRoots = objStore.listFSRoots();
+ if (hdfsRoots != null) {
+ System.out.println("HiveMetaTool:Listing FS Roots..");
+ for (String s : hdfsRoots) {
+ System.out.println(s);
+ }
+ } else {
+ System.err.println("HiveMetaTool:Encountered error during listFSRoot - " +
+ "commit of JDO transaction failed");
+ }
+ }
+
+ private void executeJDOQLSelect(String query) {
+ Collection> result = objStore.executeJDOQLSelect(query);
+ if (result != null) {
+ Iterator> iter = result.iterator();
+ while (iter.hasNext()) {
+ Object o = iter.next();
+ System.out.println(o.toString());
+ }
+ } else {
+ System.err.println("HiveMetaTool:Encountered error during executeJDOQLSelect -" +
+ "commit of JDO transaction failed.");
+ }
+ }
+
+ private void executeJDOQLUpdate(String query) {
+ long numUpdated = objStore.executeJDOQLUpdate(query);
+ if (numUpdated >= 0) {
+ System.out.println("HiveMetaTool:Number of records updated: " + numUpdated);
+ } else {
+ System.err.println("HiveMetaTool:Encountered error during executeJDOQL -" +
+ "commit of JDO transaction failed.");
+ }
+ }
+
+ private void printUpdateLocations(HashMap updateLocations) {
+ for (String key: updateLocations.keySet()) {
+ String value = updateLocations.get(key);
+ System.out.println("current location: " + key + " new location: " + value);
+ }
+ }
+
+ private void updateFSRootLocation(URI oldURI, URI newURI, boolean dryRun) {
+ HashMap updateLocations = new HashMap();
+ int count = objStore.updateFSRootLocation(oldURI, newURI, updateLocations, dryRun);
+ if (count == -1) {
+ System.err.println("HiveMetaTool:Encountered error while executing updateFSRootLocation - " +
+ "commit of JDO transaction failed, failed to update FS Root locations.");
+ } else {
+ if (!dryRun) {
+ System.out.println("HiveMetaTool: Successfully updated " + count + "FS Root locations");
+ } else {
+ printUpdateLocations(updateLocations);
+ }
+ }
+ }
+
+ public static void main(String[] args) {
+
+ HiveConf hiveConf = new HiveConf(HiveMetaTool.class);
+ HiveMetaTool metaTool = new HiveMetaTool();
+ metaTool.init();
+ CommandLineParser parser = new GnuParser();
+ CommandLine line = null;
+
+ try {
+ try {
+ line = parser.parse(metaTool.cmdLineOptions, args);
+ } catch (ParseException e) {
+ System.err.println("HiveMetaTool:Parsing failed. Reason: " + e.getLocalizedMessage());
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("metatool", metaTool.cmdLineOptions);
+ System.exit(1);
+ }
+
+ if (line.hasOption("help")) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("metatool", metaTool.cmdLineOptions);
+ } else if (line.hasOption("listFSRoot")) {
+ metaTool.initObjectStore(hiveConf);
+ metaTool.listFSRoot();
+ } else if (line.hasOption("executeJDOQL")) {
+ String query = line.getOptionValue("executeJDOQL");
+ metaTool.initObjectStore(hiveConf);
+ if (query.toLowerCase().trim().startsWith("select")) {
+ metaTool.executeJDOQLSelect(query);
+ } else if (query.toLowerCase().trim().startsWith("update")) {
+ metaTool.executeJDOQLUpdate(query);
+ } else {
+ System.err.println("HiveMetaTool:Unsupported statement type");
+ }
+ } else if (line.hasOption("updateLocation")) {
+ String[] loc = line.getOptionValues("updateLocation");
+ boolean dryRun = false;
+
+ if (loc.length != 2 && loc.length != 3) {
+ System.err.println("HiveMetaTool:updateLocation takes in 2 required and 1 " +
+ "optional arguements but " +
+ "was passed " + loc.length + " arguements");
+ }
+
+ Path newPath = new Path(loc[0]);
+ Path oldPath = new Path(loc[1]);
+
+ URI oldURI = oldPath.toUri();
+ URI newURI = newPath.toUri();
+
+ if (line.hasOption("dryRun")) {
+ dryRun = true;
+ }
+
+ /*
+ * validate input - if the old uri contains a valid port, the new uri
+ * should contain a valid port as well. Both new and old uri should
+ * contain valid host names and valid schemes.
+ */
+ if (oldURI.getHost() == null || newURI.getHost() == null) {
+ System.err.println("HiveMetaTool:A valid host is required in both old-loc and new-loc");
+ } else if (oldURI.getPort() > 0 && newURI.getPort() < 0) {
+ System.err.println("HiveMetaTool:old-loc has a valid port, new-loc should " +
+ "also contain a valid port");
+ } else if (oldURI.getScheme() == null || newURI.getScheme() == null) {
+ System.err.println("HiveMetaTool:A valid scheme is required in both old-loc and new-loc");
+ } else {
+ metaTool.initObjectStore(hiveConf);
+ metaTool.updateFSRootLocation(oldURI, newURI, dryRun);
+ }
+ } else {
+ System.err.print("HiveMetaTool:Invalid option:" + line.getOptions());
+ for (String s : line.getArgs()) {
+ System.err.print(s + " ");
+ }
+ System.err.println();
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("metatool", metaTool.cmdLineOptions);
+ }
+ } finally {
+ metaTool.shutdownObjectStore();
+ }
+ }
+}
diff --git metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
index 369eebe..925938d 100644
--- metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
+++ metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
@@ -1573,161 +1573,163 @@ public abstract class TestHiveMetaStore extends TestCase {
* Tests for list partition by filter functionality.
* @throws Exception
*/
- public void testPartitionFilter() throws Exception {
- String dbName = "filterdb";
- String tblName = "filtertbl";
- List vals = new ArrayList(3);
- vals.add("p11");
- vals.add("p21");
- vals.add("p31");
- List vals2 = new ArrayList(3);
- vals2.add("p11");
- vals2.add("p22");
- vals2.add("p31");
- List vals3 = new ArrayList(3);
- vals3.add("p12");
- vals3.add("p21");
- vals3.add("p31");
- List vals4 = new ArrayList(3);
- vals4.add("p12");
- vals4.add("p23");
- vals4.add("p31");
- List vals5 = new ArrayList(3);
- vals5.add("p13");
- vals5.add("p24");
- vals5.add("p31");
- List vals6 = new ArrayList(3);
- vals6.add("p13");
- vals6.add("p25");
- vals6.add("p31");
-
- silentDropDatabase(dbName);
-
- Database db = new Database();
- db.setName(dbName);
- client.createDatabase(db);
-
- ArrayList cols = new ArrayList(2);
- cols.add(new FieldSchema("c1", Constants.STRING_TYPE_NAME, ""));
- cols.add(new FieldSchema("c2", Constants.INT_TYPE_NAME, ""));
+ public void testPartitionFilter() throws Exception {
+ String dbName = "filterdb";
+ String tblName = "filtertbl";
+
+ List vals = new ArrayList(3);
+ vals.add("p11");
+ vals.add("p21");
+ vals.add("p31");
+ List vals2 = new ArrayList(3);
+ vals2.add("p11");
+ vals2.add("p22");
+ vals2.add("p31");
+ List vals3 = new ArrayList(3);
+ vals3.add("p12");
+ vals3.add("p21");
+ vals3.add("p31");
+ List vals4 = new ArrayList(3);
+ vals4.add("p12");
+ vals4.add("p23");
+ vals4.add("p31");
+ List vals5 = new ArrayList(3);
+ vals5.add("p13");
+ vals5.add("p24");
+ vals5.add("p31");
+ List vals6 = new ArrayList(3);
+ vals6.add("p13");
+ vals6.add("p25");
+ vals6.add("p31");
+
+ silentDropDatabase(dbName);
- ArrayList partCols = new ArrayList(3);
- partCols.add(new FieldSchema("p1", Constants.STRING_TYPE_NAME, ""));
- partCols.add(new FieldSchema("p2", Constants.STRING_TYPE_NAME, ""));
- partCols.add(new FieldSchema("p3", Constants.INT_TYPE_NAME, ""));
+ Database db = new Database();
+ db.setName(dbName);
+ client.createDatabase(db);
- Table tbl = new Table();
- tbl.setDbName(dbName);
- tbl.setTableName(tblName);
- StorageDescriptor sd = new StorageDescriptor();
- tbl.setSd(sd);
- sd.setCols(cols);
- sd.setCompressed(false);
- sd.setNumBuckets(1);
- sd.setParameters(new HashMap());
- sd.setBucketCols(new ArrayList());
- sd.setSerdeInfo(new SerDeInfo());
- sd.getSerdeInfo().setName(tbl.getTableName());
- sd.getSerdeInfo().setParameters(new HashMap());
- sd.getSerdeInfo().getParameters()
- .put(Constants.SERIALIZATION_FORMAT, "1");
- sd.setSortCols(new ArrayList());
+ ArrayList cols = new ArrayList(2);
+ cols.add(new FieldSchema("c1", Constants.STRING_TYPE_NAME, ""));
+ cols.add(new FieldSchema("c2", Constants.INT_TYPE_NAME, ""));
- tbl.setPartitionKeys(partCols);
- client.createTable(tbl);
+ ArrayList partCols = new ArrayList(3);
+ partCols.add(new FieldSchema("p1", Constants.STRING_TYPE_NAME, ""));
+ partCols.add(new FieldSchema("p2", Constants.STRING_TYPE_NAME, ""));
+ partCols.add(new FieldSchema("p3", Constants.INT_TYPE_NAME, ""));
- tbl = client.getTable(dbName, tblName);
+ Table tbl = new Table();
+ tbl.setDbName(dbName);
+ tbl.setTableName(tblName);
+ StorageDescriptor sd = new StorageDescriptor();
+ tbl.setSd(sd);
+ sd.setCols(cols);
+ sd.setCompressed(false);
+ sd.setNumBuckets(1);
+ sd.setParameters(new HashMap());
+ sd.setBucketCols(new ArrayList());
+ sd.setSerdeInfo(new SerDeInfo());
+ sd.getSerdeInfo().setName(tbl.getTableName());
+ sd.getSerdeInfo().setParameters(new HashMap());
+ sd.getSerdeInfo().getParameters()
+ .put(Constants.SERIALIZATION_FORMAT, "1");
+ sd.setSortCols(new ArrayList());
- add_partition(client, tbl, vals, "part1");
- add_partition(client, tbl, vals2, "part2");
- add_partition(client, tbl, vals3, "part3");
- add_partition(client, tbl, vals4, "part4");
- add_partition(client, tbl, vals5, "part5");
- add_partition(client, tbl, vals6, "part6");
+ tbl.setPartitionKeys(partCols);
+ client.createTable(tbl);
- checkFilter(client, dbName, tblName, "p1 = \"p11\"", 2);
- checkFilter(client, dbName, tblName, "p1 = \"p12\"", 2);
- checkFilter(client, dbName, tblName, "p2 = \"p21\"", 2);
- checkFilter(client, dbName, tblName, "p2 = \"p23\"", 1);
- checkFilter(client, dbName, tblName, "p1 = \"p11\" and p2=\"p22\"", 1);
- checkFilter(client, dbName, tblName, "p1 = \"p11\" or p2=\"p23\"", 3);
- checkFilter(client, dbName, tblName, "p1 = \"p11\" or p1=\"p12\"", 4);
-
- checkFilter(client, dbName, tblName,
- "p1 = \"p11\" or (p1=\"p12\" and p2=\"p21\")", 3);
- checkFilter(client, dbName, tblName,
- "p1 = \"p11\" or (p1=\"p12\" and p2=\"p21\") Or " +
- "(p1=\"p13\" aNd p2=\"p24\")", 4);
- //test for and or precedence
- checkFilter(client, dbName, tblName,
- "p1=\"p12\" and (p2=\"p27\" Or p2=\"p21\")", 1);
- checkFilter(client, dbName, tblName,
- "p1=\"p12\" and p2=\"p27\" Or p2=\"p21\"", 2);
-
- checkFilter(client, dbName, tblName, "p1 > \"p12\"", 2);
- checkFilter(client, dbName, tblName, "p1 >= \"p12\"", 4);
- checkFilter(client, dbName, tblName, "p1 < \"p12\"", 2);
- checkFilter(client, dbName, tblName, "p1 <= \"p12\"", 4);
- checkFilter(client, dbName, tblName, "p1 <> \"p12\"", 4);
- checkFilter(client, dbName, tblName, "p1 like \"p1.*\"", 6);
- checkFilter(client, dbName, tblName, "p2 like \"p.*3\"", 1);
-
- //Test for setting the maximum partition count
- List partitions = client.listPartitionsByFilter(dbName,
- tblName, "p1 >= \"p12\"", (short) 2);
- assertEquals("User specified row limit for partitions",
- 2, partitions.size());
+ tbl = client.getTable(dbName, tblName);
+
+ add_partition(client, tbl, vals, "part1");
+ add_partition(client, tbl, vals2, "part2");
+ add_partition(client, tbl, vals3, "part3");
+ add_partition(client, tbl, vals4, "part4");
+ add_partition(client, tbl, vals5, "part5");
+ add_partition(client, tbl, vals6, "part6");
+
+ checkFilter(client, dbName, tblName, "p1 = \"p11\"", 2);
+ checkFilter(client, dbName, tblName, "p1 = \"p12\"", 2);
+ checkFilter(client, dbName, tblName, "p2 = \"p21\"", 2);
+ checkFilter(client, dbName, tblName, "p2 = \"p23\"", 1);
+ checkFilter(client, dbName, tblName, "p1 = \"p11\" and p2=\"p22\"", 1);
+ checkFilter(client, dbName, tblName, "p1 = \"p11\" or p2=\"p23\"", 3);
+ checkFilter(client, dbName, tblName, "p1 = \"p11\" or p1=\"p12\"", 4);
+
+ checkFilter(client, dbName, tblName,
+ "p1 = \"p11\" or (p1=\"p12\" and p2=\"p21\")", 3);
+ checkFilter(client, dbName, tblName,
+ "p1 = \"p11\" or (p1=\"p12\" and p2=\"p21\") Or " +
+ "(p1=\"p13\" aNd p2=\"p24\")", 4);
+ //test for and or precedence
+ checkFilter(client, dbName, tblName,
+ "p1=\"p12\" and (p2=\"p27\" Or p2=\"p21\")", 1);
+ checkFilter(client, dbName, tblName,
+ "p1=\"p12\" and p2=\"p27\" Or p2=\"p21\"", 2);
+
+ checkFilter(client, dbName, tblName, "p1 > \"p12\"", 2);
+ checkFilter(client, dbName, tblName, "p1 >= \"p12\"", 4);
+ checkFilter(client, dbName, tblName, "p1 < \"p12\"", 2);
+ checkFilter(client, dbName, tblName, "p1 <= \"p12\"", 4);
+ checkFilter(client, dbName, tblName, "p1 <> \"p12\"", 4);
+ checkFilter(client, dbName, tblName, "p1 like \"p1.*\"", 6);
+ checkFilter(client, dbName, tblName, "p2 like \"p.*3\"", 1);
+
+ //Test for setting the maximum partition count
+ List partitions = client.listPartitionsByFilter(dbName,
+ tblName, "p1 >= \"p12\"", (short) 2);
+ assertEquals("User specified row limit for partitions",
+ 2, partitions.size());
- //Negative tests
- Exception me = null;
- try {
- client.listPartitionsByFilter(dbName,
- tblName, "p3 >= \"p12\"", (short) -1);
- } catch(MetaException e) {
- me = e;
- }
- assertNotNull(me);
- assertTrue("Filter on int partition key", me.getMessage().contains(
- "Filtering is supported only on partition keys of type string"));
+ //Negative tests
+ Exception me = null;
+ try {
+ client.listPartitionsByFilter(dbName,
+ tblName, "p3 >= \"p12\"", (short) -1);
+ } catch(MetaException e) {
+ me = e;
+ }
+ assertNotNull(me);
+ assertTrue("Filter on int partition key", me.getMessage().contains(
+ "Filtering is supported only on partition keys of type string"));
- me = null;
- try {
- client.listPartitionsByFilter(dbName,
- tblName, "c1 >= \"p12\"", (short) -1);
- } catch(MetaException e) {
- me = e;
- }
- assertNotNull(me);
- assertTrue("Filter on invalid key", me.getMessage().contains(
- " is not a partitioning key for the table"));
+ me = null;
+ try {
+ client.listPartitionsByFilter(dbName,
+ tblName, "c1 >= \"p12\"", (short) -1);
+ } catch(MetaException e) {
+ me = e;
+ }
+ assertNotNull(me);
+ assertTrue("Filter on invalid key", me.getMessage().contains(
+ " is not a partitioning key for the table"));
- me = null;
- try {
- client.listPartitionsByFilter(dbName,
- tblName, "c1 >= ", (short) -1);
- } catch(MetaException e) {
- me = e;
- }
- assertNotNull(me);
- assertTrue("Invalid filter string", me.getMessage().contains(
- "Error parsing partition filter"));
+ me = null;
+ try {
+ client.listPartitionsByFilter(dbName,
+ tblName, "c1 >= ", (short) -1);
+ } catch(MetaException e) {
+ me = e;
+ }
+ assertNotNull(me);
+ assertTrue("Invalid filter string", me.getMessage().contains(
+ "Error parsing partition filter"));
- me = null;
- try {
- client.listPartitionsByFilter("invDBName",
- "invTableName", "p1 = \"p11\"", (short) -1);
- } catch(NoSuchObjectException e) {
- me = e;
- }
- assertNotNull(me);
- assertTrue("NoSuchObject exception", me.getMessage().contains(
- "database/table does not exist"));
+ me = null;
+ try {
+ client.listPartitionsByFilter("invDBName",
+ "invTableName", "p1 = \"p11\"", (short) -1);
+ } catch(NoSuchObjectException e) {
+ me = e;
+ }
+ assertNotNull(me);
+ assertTrue("NoSuchObject exception", me.getMessage().contains(
+ "database/table does not exist"));
- client.dropTable(dbName, tblName);
- client.dropDatabase(dbName);
+ client.dropTable(dbName, tblName);
+ client.dropDatabase(dbName);
}
+
/**
* Test filtering on table with single partition
* @throws Exception
@@ -2299,5 +2301,4 @@ public abstract class TestHiveMetaStore extends TestCase {
createPartitions(dbName, tbl, values);
}
-
}
diff --git metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java
new file mode 100644
index 0000000..5790ae9
--- /dev/null
+++ metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java
@@ -0,0 +1,224 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.io.ByteArrayOutputStream;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.Type;
+import org.apache.hadoop.hive.metastore.tools.HiveMetaTool;
+import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.util.StringUtils;
+
+public class TestHiveMetaTool extends TestCase {
+
+ private HiveMetaStoreClient client;
+
+ private PrintStream originalOut;
+ private OutputStream os;
+ private PrintStream ps;
+ private String locationUri;
+
+
+ private void dropDatabase(String dbName) throws Exception {
+ try {
+ client.dropDatabase(dbName);
+ } catch (NoSuchObjectException e) {
+ } catch (InvalidOperationException e) {
+ } catch (Exception e) {
+ throw e;
+ }
+ }
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+
+ try {
+ HiveConf hiveConf = new HiveConf(HiveMetaTool.class);
+ client = new HiveMetaStoreClient(hiveConf, null);
+
+ // Setup output stream to redirect output to
+ os = new ByteArrayOutputStream();
+ ps = new PrintStream(os);
+
+ // create a dummy database and a couple of dummy tables
+ String dbName = "testDB";
+ String typeName = "Person";
+ String tblName = "simpleTbl";
+
+ Database db = new Database();
+ db.setName(dbName);
+ client.dropTable(dbName, tblName);
+ dropDatabase(dbName);
+ client.createDatabase(db);
+ locationUri = db.getLocationUri();
+
+ client.dropType(typeName);
+ Type typ1 = new Type();
+ typ1.setName(typeName);
+ typ1.setFields(new ArrayList(2));
+ typ1.getFields().add(
+ new FieldSchema("name", Constants.STRING_TYPE_NAME, ""));
+ typ1.getFields().add(
+ new FieldSchema("income", Constants.INT_TYPE_NAME, ""));
+ client.createType(typ1);
+
+ Table tbl = new Table();
+ tbl.setDbName(dbName);
+ tbl.setTableName(tblName);
+ StorageDescriptor sd = new StorageDescriptor();
+ tbl.setSd(sd);
+ sd.setCols(typ1.getFields());
+ sd.setCompressed(false);
+ sd.setNumBuckets(1);
+ sd.setParameters(new HashMap());
+ sd.getParameters().put("test_param_1", "Use this for comments etc");
+ sd.setBucketCols(new ArrayList(2));
+ sd.getBucketCols().add("name");
+ sd.setSerdeInfo(new SerDeInfo());
+ sd.getSerdeInfo().setName(tbl.getTableName());
+ sd.getSerdeInfo().setParameters(new HashMap());
+ sd.getSerdeInfo().getParameters().put(
+ org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "1");
+ sd.getSerdeInfo().setSerializationLib(
+ org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
+ tbl.setPartitionKeys(new ArrayList());
+
+ client.createTable(tbl);
+ client.close();
+ } catch (Exception e) {
+ System.err.println("Unable to setup the hive metatool test");
+ System.err.println(StringUtils.stringifyException(e));
+ throw new Exception(e);
+ }
+ }
+
+ private void redirectOutputStream() {
+
+ originalOut = System.out;
+ System.setOut(ps);
+
+ }
+
+ private void restoreOutputStream() {
+
+ System.setOut(originalOut);
+ }
+
+ public void testListFSRoot() throws Exception {
+
+ redirectOutputStream();
+ String[] args = new String[1];
+ args[0] = new String("-listFSRoot");
+
+ try {
+ HiveMetaTool.main(args);
+ String out = os.toString();
+ boolean b = out.contains(locationUri);
+ assertTrue(b);
+ } finally {
+ restoreOutputStream();
+ System.out.println("Completed testListFSRoot");
+ }
+ }
+
+ public void testExecuteJDOQL() throws Exception {
+
+ redirectOutputStream();
+ String[] args = new String[2];
+ args[0] = new String("-executeJDOQL");
+ args[1] = new String("select locationUri from org.apache.hadoop.hive.metastore.model.MDatabase");
+
+ try {
+ HiveMetaTool.main(args);
+ String out = os.toString();
+ boolean b = out.contains(locationUri);
+ assertTrue(b);
+ } finally {
+ restoreOutputStream();
+ System.out.println("Completed testExecuteJDOQL");
+ }
+ }
+
+ public void testUpdateFSRootLocation() throws Exception {
+
+ redirectOutputStream();
+ String newLocationUri = "hdfs://nn-ha-uri/user/hive/warehouse";
+ String[] args = new String[3];
+ args[0] = new String("-updateLocation");
+ args[1] = new String(newLocationUri);
+ args[2] = new String(locationUri);
+
+ String[] args2 = new String[1];
+ args2[0] = new String("-listFSRoot");
+
+ try {
+
+ // perform HA upgrade
+ HiveMetaTool.main(args);
+
+ // obtain new HDFS root
+ HiveMetaTool.main(args2);
+
+ String out = os.toString();
+ boolean b = out.contains(newLocationUri);
+
+ if (b) {
+ System.out.println("updateFSRootLocation successful");
+ } else {
+ System.out.println("updateFSRootLocation failed");
+ }
+ // restore the original HDFS root if needed
+ if (b) {
+ args[1] = new String(locationUri);
+ args[2] = new String(newLocationUri);
+ HiveMetaTool.main(args);
+ }
+ } finally {
+ restoreOutputStream();
+ System.out.println("Completed testUpdateFSRootLocation..");
+ }
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ try {
+ super.tearDown();
+
+ } catch (Throwable e) {
+ System.err.println("Unable to close metastore");
+ System.err.println(StringUtils.stringifyException(e));
+ throw new Exception(e);
+ }
+ }
+}