Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1055171) +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -164,6 +164,8 @@ // CLI CLIIGNOREERRORS("hive.cli.errors.ignore", false), + HIVE_METASTORE_FS_HANDLER_CLS("hive.metastore.fs.handler.class", "org.apache.hadoop.hive.metastore.HiveMetaStoreFsImpl"), + // Things we log in the jobconf // session identifier Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java (revision 0) +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java (revision 0) @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import java.io.FileNotFoundException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.Trash; +import org.apache.hadoop.hive.metastore.api.MetaException; + +public class HiveMetaStoreFsImpl implements MetaStoreFS { + + public static final Log LOG = LogFactory + .getLog("hive.metastore.hivemetastoressimpl"); + + @Override + public boolean deleteDir(FileSystem fs, Path f, boolean recursive, + Configuration conf) throws MetaException { + LOG.info("deleting " + f); + + // older versions of Hadoop don't have a Trash constructor based on the + // Path or FileSystem. So need to achieve this by creating a dummy conf. + // this needs to be filtered out based on version + Configuration dupConf = new Configuration(conf); + FileSystem.setDefaultUri(dupConf, fs.getUri()); + + try { + Trash trashTmp = new Trash(dupConf); + if (trashTmp.moveToTrash(f)) { + LOG.info("Moved to trash: " + f); + return true; + } + + if (fs.delete(f, true)) { + LOG.info("Deleted the diretory " + f); + return true; + } + + if (fs.exists(f)) { + throw new MetaException("Unable to delete directory: " + f); + } + } catch (FileNotFoundException e) { + return true; // ok even if there is not data + } catch (Exception e) { + Warehouse.closeFs(fs); + MetaStoreUtils.logAndThrowMetaException(e); + } + return false; + } + +} Index: metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java (revision 0) +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java (revision 0) @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.metastore.api.MetaException; + +/** + * Define a set of APIs that may vary in different environments + */ +public interface MetaStoreFS { + + /** + * delete a directory + * + * @param f + * @param recursive + * @return + * @throws MetaException + */ + public boolean deleteDir(FileSystem fs, Path f, boolean recursive, + Configuration conf) throws MetaException; + +} Index: metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java (revision 1055171) +++ metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java (working copy) @@ -37,11 +37,12 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.Trash; import org.apache.hadoop.hive.common.FileUtils; +import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.util.ReflectionUtils; /** * This class represents a warehouse where data of Hive tables is stored @@ -54,6 +55,8 @@ private static final String DATABASE_WAREHOUSE_SUFFIX = ".db"; public static final Log LOG = LogFactory.getLog("hive.metastore.warehouse"); + + private MetaStoreFS fsHandler = null; public Warehouse(Configuration conf) throws MetaException { this.conf = conf; @@ -62,8 +65,26 @@ throw new MetaException(HiveConf.ConfVars.METASTOREWAREHOUSE.varname + " is not set in the config or blank"); } + fsHandler = getMetaStoreFsHandler(conf); + } + + private MetaStoreFS getMetaStoreFsHandler(Configuration conf) + throws MetaException { + String handlerClassStr = HiveConf.getVar(conf, + HiveConf.ConfVars.HIVE_METASTORE_FS_HANDLER_CLS); + try { + Class handlerClass = (Class) Class + .forName(handlerClassStr, true, JavaUtils.getClassLoader()); + MetaStoreFS handler = (MetaStoreFS) ReflectionUtils.newInstance( + handlerClass, conf); + return handler; + } catch (ClassNotFoundException e) { + throw new MetaException("Error in loading index handler." + + e.getMessage()); + } } + /** * Helper functions to convert IOException to MetaException */ @@ -146,39 +167,8 @@ } public boolean deleteDir(Path f, boolean recursive) throws MetaException { - LOG.info("deleting " + f); - FileSystem fs = null; - try { - fs = getFs(f); - if (!fs.exists(f)) { - return false; - } - - // older versions of Hadoop don't have a Trash constructor based on the - // Path or FileSystem. So need to achieve this by creating a dummy conf. - // this needs to be filtered out based on version - Configuration dupConf = new Configuration(conf); - FileSystem.setDefaultUri(dupConf, fs.getUri()); - - Trash trashTmp = new Trash(dupConf); - if (trashTmp.moveToTrash(f)) { - LOG.info("Moved to trash: " + f); - return true; - } - if (fs.delete(f, true)) { - LOG.info("Deleted the diretory " + f); - return true; - } - if (fs.exists(f)) { - throw new MetaException("Unable to delete directory: " + f); - } - } catch (FileNotFoundException e) { - return true; // ok even if there is not data - } catch (IOException e) { - closeFs(fs); - MetaStoreUtils.logAndThrowMetaException(e); - } - return false; + FileSystem fs = getFs(f); + return fsHandler.deleteDir(fs, f, recursive, conf); } /*