diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java index a319b88..4642ec2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java @@ -30,8 +30,13 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.FileUtils; +import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.io.HdfsUtils; import org.apache.hadoop.hive.metastore.HiveMetaHookLoader; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; @@ -56,6 +61,8 @@ import org.apache.hadoop.hive.metastore.api.UnknownDBException; import org.apache.hadoop.hive.metastore.api.UnknownTableException; import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.shims.HadoopShims; +import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.ql.stats.StatsUtils; import org.apache.thrift.TException; @@ -115,6 +122,18 @@ protected void drop_table_with_environment_context(String dbname, String name, } @Override + public void truncateTable(String dbName, String tableName, List partNames) throws MetaException, TException { + // First try temp table + org.apache.hadoop.hive.metastore.api.Table table = getTempTable(dbName, tableName); + if (table != null) { + truncateTempTable(table); + return; + } + // Try underlying client + super.truncateTable(dbName, tableName, partNames); + } + + @Override public org.apache.hadoop.hive.metastore.api.Table getTable(String dbname, String name) throws MetaException, TException, NoSuchObjectException { // First check temp tables @@ -509,6 +528,63 @@ private static boolean fieldSchemaEqualsIgnoreComment(FieldSchema left, FieldSch return false; } + private boolean needToUpdateStats(Map props, EnvironmentContext environmentContext) { + if (null == props) { + return false; + } + boolean statsPresent = false; + for (String stat : StatsSetupConst.supportedStats) { + String statVal = props.get(stat); + if (statVal != null && Long.parseLong(statVal) > 0) { + statsPresent = true; + //In the case of truncate table, we set the stats to be 0. + props.put(stat, "0"); + } + } + //first set basic stats to true + StatsSetupConst.setBasicStatsState(props, StatsSetupConst.TRUE); + environmentContext.putToProperties(StatsSetupConst.STATS_GENERATED, StatsSetupConst.TASK); + //then invalidate column stats + StatsSetupConst.clearColumnStatsState(props); + return statsPresent; + } + + private void truncateTempTable(org.apache.hadoop.hive.metastore.api.Table table) throws MetaException, TException { + + boolean isAutopurge = "true".equalsIgnoreCase(table.getParameters().get("auto.purge")); + try { + // this is not transactional + Path location = new Path(table.getSd().getLocation()); + + FileSystem fs = location.getFileSystem(conf); + HadoopShims.HdfsEncryptionShim shim + = ShimLoader.getHadoopShims().createHdfsEncryptionShim(fs, conf); + if (!shim.isPathEncrypted(location)) { + HdfsUtils.HadoopFileStatus status = new HdfsUtils.HadoopFileStatus(conf, fs, location); + FileStatus targetStatus = fs.getFileStatus(location); + String targetGroup = targetStatus == null ? null : targetStatus.getGroup(); + FileUtils.moveToTrash(fs, location, conf, isAutopurge); + fs.mkdirs(location); + HdfsUtils.setFullFileStatus(conf, status, targetGroup, fs, location, false); + } else { + FileStatus[] statuses = fs.listStatus(location, FileUtils.HIDDEN_FILES_PATH_FILTER); + if ((statuses != null) && (statuses.length > 0)) { + boolean success = Hive.trashFiles(fs, statuses, conf, isAutopurge); + if (!success) { + throw new HiveException("Error in deleting the contents of " + location.toString()); + } + } + } + + EnvironmentContext environmentContext = new EnvironmentContext(); + if (needToUpdateStats(table.getParameters(), environmentContext)) { + alter_table_with_environmentContext(table.getDbName(), table.getTableName(), table, environmentContext); + } + } catch (Exception e) { + throw new MetaException(e.getMessage()); + } + } + private void dropTempTable(org.apache.hadoop.hive.metastore.api.Table table, boolean deleteData, EnvironmentContext envContext) throws MetaException, TException, NoSuchObjectException, UnsupportedOperationException { diff --git a/ql/src/test/queries/clientpositive/temp_table_truncate.q b/ql/src/test/queries/clientpositive/temp_table_truncate.q new file mode 100644 index 0000000..7b0fefc --- /dev/null +++ b/ql/src/test/queries/clientpositive/temp_table_truncate.q @@ -0,0 +1,21 @@ +set hive.mapred.mode=nonstrict; +set hive.explain.user=false; +CREATE TEMPORARY TABLE tmp_src AS SELECT * FROM src WHERE key % 2 = 0; +CREATE TEMPORARY TABLE tmp_srcpart AS SELECT * FROM srcpart; + +DESCRIBE tmp_src; +DESCRIBE tmp_srcpart; +SHOW TABLES; + +SELECT count(*) FROM tmp_src; +SELECT count(*) FROM tmp_srcpart; + +EXPLAIN TRUNCATE TABLE tmp_src; +TRUNCATE TABLE tmp_src; + +SELECT count(*) FROM tmp_src; + +EXPLAIN TRUNCATE TABLE tmp_srcpart; +TRUNCATE TABLE tmp_srcpart; + +SELECT count(*) FROM tmp_srcpart; \ No newline at end of file diff --git a/ql/src/test/results/clientpositive/temp_table_truncate.q.out b/ql/src/test/results/clientpositive/temp_table_truncate.q.out new file mode 100644 index 0000000..b1af432 --- /dev/null +++ b/ql/src/test/results/clientpositive/temp_table_truncate.q.out @@ -0,0 +1,143 @@ +PREHOOK: query: CREATE TEMPORARY TABLE tmp_src AS SELECT * FROM src WHERE key % 2 = 0 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@tmp_src +POSTHOOK: query: CREATE TEMPORARY TABLE tmp_src AS SELECT * FROM src WHERE key % 2 = 0 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tmp_src +PREHOOK: query: CREATE TEMPORARY TABLE tmp_srcpart AS SELECT * FROM srcpart +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: database:default +PREHOOK: Output: default@tmp_srcpart +POSTHOOK: query: CREATE TEMPORARY TABLE tmp_srcpart AS SELECT * FROM srcpart +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tmp_srcpart +PREHOOK: query: DESCRIBE tmp_src +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@tmp_src +POSTHOOK: query: DESCRIBE tmp_src +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@tmp_src +key string +value string +PREHOOK: query: DESCRIBE tmp_srcpart +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@tmp_srcpart +POSTHOOK: query: DESCRIBE tmp_srcpart +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@tmp_srcpart +key string +value string +ds string +hr string +PREHOOK: query: SHOW TABLES +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: SHOW TABLES +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +alltypesorc +cbo_t1 +cbo_t2 +cbo_t3 +lineitem +part +src +src1 +src_cbo +src_json +src_sequencefile +src_thrift +srcbucket +srcbucket2 +srcpart +tmp_src +tmp_srcpart +PREHOOK: query: SELECT count(*) FROM tmp_src +PREHOOK: type: QUERY +PREHOOK: Input: default@tmp_src +#### A masked pattern was here #### +POSTHOOK: query: SELECT count(*) FROM tmp_src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tmp_src +#### A masked pattern was here #### +247 +PREHOOK: query: SELECT count(*) FROM tmp_srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@tmp_srcpart +#### A masked pattern was here #### +POSTHOOK: query: SELECT count(*) FROM tmp_srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tmp_srcpart +#### A masked pattern was here #### +2000 +PREHOOK: query: EXPLAIN TRUNCATE TABLE tmp_src +PREHOOK: type: TRUNCATETABLE +POSTHOOK: query: EXPLAIN TRUNCATE TABLE tmp_src +POSTHOOK: type: TRUNCATETABLE +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Truncate Table Operator: + Truncate Table or Partition + TableName: tmp_src + +PREHOOK: query: TRUNCATE TABLE tmp_src +PREHOOK: type: TRUNCATETABLE +PREHOOK: Output: default@tmp_src +POSTHOOK: query: TRUNCATE TABLE tmp_src +POSTHOOK: type: TRUNCATETABLE +POSTHOOK: Output: default@tmp_src +PREHOOK: query: SELECT count(*) FROM tmp_src +PREHOOK: type: QUERY +PREHOOK: Input: default@tmp_src +#### A masked pattern was here #### +POSTHOOK: query: SELECT count(*) FROM tmp_src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tmp_src +#### A masked pattern was here #### +0 +PREHOOK: query: EXPLAIN TRUNCATE TABLE tmp_srcpart +PREHOOK: type: TRUNCATETABLE +POSTHOOK: query: EXPLAIN TRUNCATE TABLE tmp_srcpart +POSTHOOK: type: TRUNCATETABLE +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Truncate Table Operator: + Truncate Table or Partition + TableName: tmp_srcpart + +PREHOOK: query: TRUNCATE TABLE tmp_srcpart +PREHOOK: type: TRUNCATETABLE +PREHOOK: Output: default@tmp_srcpart +POSTHOOK: query: TRUNCATE TABLE tmp_srcpart +POSTHOOK: type: TRUNCATETABLE +POSTHOOK: Output: default@tmp_srcpart +PREHOOK: query: SELECT count(*) FROM tmp_srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@tmp_srcpart +#### A masked pattern was here #### +POSTHOOK: query: SELECT count(*) FROM tmp_srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tmp_srcpart +#### A masked pattern was here #### +0