diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 604bea7..0f98d6a 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -384,6 +384,10 @@ METADATA_EXPORT_LOCATION("hive.metadata.export.location", ""), MOVE_EXPORTED_METADATA_TO_TRASH("hive.metadata.move.exported.metadata.to.trash", true), + METASTORE_TABLES_TTL_CREATE("hive.metadata.ttl.create.seconds", -1), + METASTORE_TABLES_TTL_ACCESS("hive.metadata.ttl.access.seconds", -1), + METASTORE_TABLES_TTL_DATABASE("hive.metadata.ttl.database", ""), + // CLI CLIIGNOREERRORS("hive.cli.errors.ignore", false), CLIPRINTCURRENTDB("hive.cli.print.current.db", false), diff --git metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 4b4f4f2..5913930 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -19,12 +19,14 @@ package org.apache.hadoop.hive.metastore; import static org.apache.commons.lang.StringUtils.join; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.*; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -228,11 +230,60 @@ public void setConf(Configuration conf) { } else { LOG.info("Initialized ObjectStore"); } + startTTLChecker(hiveConf); } finally { pmfPropLock.unlock(); } } + private static Thread ttlChecker; + + private void startTTLChecker(final Configuration conf) { + if (ttlChecker != null) { + return; + } + final int create = HiveConf.getIntVar(conf, METASTORE_TABLES_TTL_CREATE); + final int access = HiveConf.getIntVar(conf, METASTORE_TABLES_TTL_ACCESS); + final String database = HiveConf.getVar(conf, METASTORE_TABLES_TTL_DATABASE); + if (create <= 0 || access <= 0) { + return; + } + ttlChecker = new Thread("TTL-Checker") { + public void run() { + while (true) { + try { + Thread.sleep(access * 1000); + } catch (InterruptedException e) { + // ignore + } + checkTTL(create, access, database); + } + } + }; + ttlChecker.setDaemon(true); + ttlChecker.start(); + } + + private void checkTTL(int create, int access, String database) { + if (create > 0 && access > 0) { + try { + for (Table table : leastRecentlyAccessedTables(create, access, database, 3)) { + String tableName = table.getDbName() + "." + table.getTableName(); + LOG.warn("Dropping table " + tableName + + " by TTL policy (Created:" + new Date(table.getCreateTime() * 1000l) + + ", LastAccessed:" + new Date(table.getLastAccessTime() * 1000l) +")"); + try { + dropTable(table.getDbName(), table.getTableName()); + } catch (Exception e) { + LOG.warn("Failed to drop table" + tableName, e); + } + } + } catch (MetaException e) { + LOG.warn("Failed to get table", e); + } + } + } + private ClassLoader classLoader; { classLoader = Thread.currentThread().getContextClassLoader(); @@ -900,6 +951,9 @@ private MTable getMTable(String db, String table) { query.setUnique(true); mtbl = (MTable) query.execute(table, db); pm.retrieve(mtbl); + if (mtbl != null) { + mtbl.setLastAccessTime(nowInSeconds()); + } commited = commitTransaction(); } finally { if (!commited) { @@ -909,6 +963,43 @@ private MTable getMTable(String db, String table) { return mtbl; } + private int nowInSeconds() { + return (int) (System.currentTimeMillis() / 1000); + } + + private List