Index: common/src/java/org/apache/hadoop/hive/common/ServerUtils.java =================================================================== --- common/src/java/org/apache/hadoop/hive/common/ServerUtils.java (revision 0) +++ common/src/java/org/apache/hadoop/hive/common/ServerUtils.java (revision 0) @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; + +/** + * + * ServerUtils. + * + */ +public class ServerUtils { + + public static final Log LOG = LogFactory.getLog(ServerUtils.class); + + public static void cleanUpScratchDir(HiveConf hiveConf) { + if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_START_CLEANUP_SCRATCHDIR)) { + String hiveScratchDir = hiveConf.get(HiveConf.ConfVars.SCRATCHDIR.varname); + try { + Path jobScratchDir = new Path(hiveScratchDir); + LOG.info("Cleaning scratchDir : " + hiveScratchDir); + FileSystem fileSystem = jobScratchDir.getFileSystem(hiveConf); + fileSystem.delete(jobScratchDir, true); + } + // Even if the cleanup throws some exception it will continue. + catch (Throwable e) { + LOG.warn("Unable to delete scratchDir : " + hiveScratchDir, e); + } + } + } + +} Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1174277) +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -478,6 +478,8 @@ // The class responsible for logging client side performance metrics // Must be a subclass of org.apache.hadoop.hive.ql.log.PerfLogger HIVE_PERF_LOGGER("hive.exec.perf.logger", "org.apache.hadoop.hive.ql.log.PerfLogger"), + // Whether to delete the scratchdir while startup + HIVE_START_CLEANUP_SCRATCHDIR("hive.start.cleanup.scratchdir", false), ; public final String varname; Index: conf/hive-default.xml =================================================================== --- conf/hive-default.xml (revision 1174277) +++ conf/hive-default.xml (working copy) @@ -1193,4 +1193,10 @@ The class responsible logging client side performance metrics. Must be a subclass of org.apache.hadoop.hive.ql.log.PerfLogger + + hive.start.cleanup.scratchdir + false + To cleanup the hive scratchdir while starting the hive server + + Index: service/src/java/org/apache/hadoop/hive/service/HiveServer.java =================================================================== --- service/src/java/org/apache/hadoop/hive/service/HiveServer.java (revision 1174277) +++ service/src/java/org/apache/hadoop/hive/service/HiveServer.java (working copy) @@ -34,6 +34,7 @@ import org.apache.commons.cli.OptionBuilder; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.common.ServerUtils; import org.apache.hadoop.hive.common.LogUtils; import org.apache.hadoop.hive.common.LogUtils.LogInitializationException; import org.apache.hadoop.hive.common.cli.CommonCliOptions; @@ -60,7 +61,8 @@ import org.apache.thrift.transport.TServerTransport; import org.apache.thrift.transport.TTransport; import org.apache.thrift.transport.TTransportFactory; - +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import com.facebook.fb303.fb_status; /** @@ -647,7 +649,7 @@ } } } - + public static void main(String[] args) { try { HiveServerCli cli = new HiveServerCli(); @@ -666,10 +668,10 @@ HiveServerHandler.LOG.warn(e.getMessage()); } + HiveConf conf = new HiveConf(HiveServerHandler.class); + ServerUtils.cleanUpScratchDir(conf); TServerTransport serverTransport = new TServerSocket(cli.port); - HiveConf conf = new HiveConf(HiveServerHandler.class); - // set all properties specified on the command line for (Map.Entry item : hiveconf.entrySet()) { conf.set((String) item.getKey(), (String) item.getValue()); Index: service/src/test/org/apache/hadoop/hive/service/TestHiveServer.java =================================================================== --- service/src/test/org/apache/hadoop/hive/service/TestHiveServer.java (revision 1174277) +++ service/src/test/org/apache/hadoop/hive/service/TestHiveServer.java (working copy) @@ -23,7 +23,9 @@ import junit.framework.TestCase; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.ServerUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Schema; @@ -390,4 +392,33 @@ } } + public void testScratchDirShouldNotClearWhileStartup() throws Exception { + FileSystem fs = FileSystem.get(conf); + Path scratchDirPath = new Path(HiveConf.getVar(conf, + HiveConf.ConfVars.SCRATCHDIR)); + boolean fileExists = fs.exists(scratchDirPath); + if (!fileExists) { + fileExists = fs.mkdirs(scratchDirPath); + } + ServerUtils.cleanUpScratchDir(conf); + assertTrue("Scratch dir is not available after startup", fs.exists(scratchDirPath)); + } + + public void testScratchDirShouldClearWhileStartup() throws Exception { + FileSystem fs = FileSystem.get(conf); + Path scratchDirPath = new Path(HiveConf.getVar(conf, + HiveConf.ConfVars.SCRATCHDIR)); + boolean fileExists = fs.exists(scratchDirPath); + if (!fileExists) { + fileExists = fs.mkdirs(scratchDirPath); + } + try { + conf.setBoolVar(HiveConf.ConfVars.HIVE_START_CLEANUP_SCRATCHDIR, true); + ServerUtils.cleanUpScratchDir(conf); + } finally { + conf.setBoolVar(HiveConf.ConfVars.HIVE_START_CLEANUP_SCRATCHDIR, false); + } + assertFalse("Scratch dir is available after startup", fs.exists(scratchDirPath)); + } + }