diff --git ql/src/java/org/apache/hadoop/hive/ql/index/AggregateIndexHandler.java ql/src/java/org/apache/hadoop/hive/ql/index/AggregateIndexHandler.java index c39e823..fdb5b58 100644 --- ql/src/java/org/apache/hadoop/hive/ql/index/AggregateIndexHandler.java +++ ql/src/java/org/apache/hadoop/hive/ql/index/AggregateIndexHandler.java @@ -23,6 +23,7 @@ import java.util.Map; import java.util.Set; +import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Index; @@ -155,7 +156,7 @@ private void createAggregationFunction(List indexTblCols, String pr builderConf.setBoolVar(HiveConf.ConfVars.HIVEMERGETEZFILES, false); Task rootTask = IndexUtils.createRootTask(builderConf, inputs, outputs, command, (LinkedHashMap) partSpec, indexTableName, dbName); - + getConf().set(StatsSetupConst.STATS_TMP_LOC, builderConf.get(StatsSetupConst.STATS_TMP_LOC)); return rootTask; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java index 0af8d7d..0c33c5f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java +++ ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java @@ -28,6 +28,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Index; @@ -285,8 +286,8 @@ public void analyzeIndexDefinition(Table baseTable, Index index, } Task rootTask = IndexUtils.createRootTask(builderConf, inputs, outputs, - command, (LinkedHashMap) partSpec, indexTableName, dbName); - + command, partSpec, indexTableName, dbName); + getConf().set(StatsSetupConst.STATS_TMP_LOC, builderConf.get(StatsSetupConst.STATS_TMP_LOC)); return rootTask; } diff --git ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java index 0135a71..4b9d61f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java +++ ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java @@ -28,6 +28,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -147,6 +148,7 @@ public void analyzeIndexDefinition(Table baseTable, Index index, builderConf.setBoolVar(HiveConf.ConfVars.HIVEMERGETEZFILES, false); Task rootTask = IndexUtils.createRootTask(builderConf, inputs, outputs, command, partSpec, indexTableName, dbName); + getConf().set(StatsSetupConst.STATS_TMP_LOC, builderConf.get(StatsSetupConst.STATS_TMP_LOC)); return rootTask; } diff --git ql/src/test/queries/clientpositive/index_auth.q ql/src/test/queries/clientpositive/index_auth.q index 413b37e..03d77f1 100644 --- ql/src/test/queries/clientpositive/index_auth.q +++ ql/src/test/queries/clientpositive/index_auth.q @@ -1,3 +1,4 @@ +set hive.stats.dbclass=fs; SET hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; create table foobar(key int, value string) PARTITIONED BY (ds string, hr string); alter table foobar add partition (ds='2008-04-08',hr='12'); diff --git ql/src/test/queries/clientpositive/index_auto.q ql/src/test/queries/clientpositive/index_auto.q index cb8a1d6..77733aa 100644 --- ql/src/test/queries/clientpositive/index_auto.q +++ ql/src/test/queries/clientpositive/index_auto.q @@ -3,6 +3,7 @@ -- without indexing SELECT key, value FROM src WHERE key > 80 AND key < 100 ORDER BY key; +set hive.stats.dbclass=fs; CREATE INDEX src_index ON TABLE src(key) as 'COMPACT' WITH DEFERRED REBUILD; ALTER INDEX src_index ON src REBUILD; @@ -25,4 +26,4 @@ SET hive.optimize.index.filter.compact.minsize=0; EXPLAIN SELECT key, value FROM src WHERE key > 80 AND key < 100 ORDER BY key; SELECT key, value FROM src WHERE key > 80 AND key < 100 ORDER BY key; -DROP INDEX src_index on src; \ No newline at end of file +DROP INDEX src_index on src; diff --git ql/src/test/queries/clientpositive/index_auto_empty.q ql/src/test/queries/clientpositive/index_auto_empty.q index cb32162..41f4a40 100644 --- ql/src/test/queries/clientpositive/index_auto_empty.q +++ ql/src/test/queries/clientpositive/index_auto_empty.q @@ -3,6 +3,7 @@ -- Create temp, and populate it with some values in src. CREATE TABLE temp(key STRING, val STRING) STORED AS TEXTFILE; +set hive.stats.dbclass=fs; -- Build an index on temp. CREATE INDEX temp_index ON TABLE temp(key) as 'COMPACT' WITH DEFERRED REBUILD; ALTER INDEX temp_index ON temp REBUILD; diff --git ql/src/test/queries/clientpositive/index_auto_file_format.q ql/src/test/queries/clientpositive/index_auto_file_format.q index 790e6c2..2967bd6 100644 --- ql/src/test/queries/clientpositive/index_auto_file_format.q +++ ql/src/test/queries/clientpositive/index_auto_file_format.q @@ -1,3 +1,4 @@ +set hive.stats.dbclass=fs; -- test automatic use of index on different file formats CREATE INDEX src_index ON TABLE src(key) as 'COMPACT' WITH DEFERRED REBUILD; ALTER INDEX src_index ON src REBUILD; @@ -16,4 +17,4 @@ SET hive.optimize.index.filter.compact.minsize=0; EXPLAIN SELECT key, value FROM src WHERE key=86 ORDER BY key; SELECT key, value FROM src WHERE key=86 ORDER BY key; -DROP INDEX src_index on src; \ No newline at end of file +DROP INDEX src_index on src; diff --git ql/src/test/queries/clientpositive/index_auto_mult_tables.q ql/src/test/queries/clientpositive/index_auto_mult_tables.q index 2bf8481..a672e06 100644 --- ql/src/test/queries/clientpositive/index_auto_mult_tables.q +++ ql/src/test/queries/clientpositive/index_auto_mult_tables.q @@ -4,6 +4,7 @@ EXPLAIN SELECT a.key, a.value FROM src a JOIN srcpart b ON (a.key = b.key) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key; SELECT a.key, a.value FROM src a JOIN srcpart b ON (a.key = b.key) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key; +set hive.stats.dbclass=fs; CREATE INDEX src_index ON TABLE src(key) as 'BITMAP' WITH DEFERRED REBUILD; ALTER INDEX src_index ON src REBUILD; diff --git ql/src/test/queries/clientpositive/index_auto_mult_tables_compact.q ql/src/test/queries/clientpositive/index_auto_mult_tables_compact.q index 808a04c..d78e0fd 100644 --- ql/src/test/queries/clientpositive/index_auto_mult_tables_compact.q +++ ql/src/test/queries/clientpositive/index_auto_mult_tables_compact.q @@ -4,6 +4,7 @@ EXPLAIN SELECT a.key, a.value FROM src a JOIN srcpart b ON (a.key = b.key) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key; SELECT a.key, a.value FROM src a JOIN srcpart b ON (a.key = b.key) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key; +set hive.stats.dbclass=fs; CREATE INDEX src_index ON TABLE src(key) as 'COMPACT' WITH DEFERRED REBUILD; ALTER INDEX src_index ON src REBUILD; diff --git ql/src/test/queries/clientpositive/index_auto_multiple.q ql/src/test/queries/clientpositive/index_auto_multiple.q index 06e97fa..f0a91b4 100644 --- ql/src/test/queries/clientpositive/index_auto_multiple.q +++ ql/src/test/queries/clientpositive/index_auto_multiple.q @@ -1,3 +1,4 @@ +set hive.stats.dbclass=fs; -- With multiple indexes, make sure we choose which to use in a consistent order CREATE INDEX src_key_index ON TABLE src(key) as 'COMPACT' WITH DEFERRED REBUILD; @@ -13,4 +14,4 @@ EXPLAIN SELECT key, value FROM src WHERE key=86 ORDER BY key; SELECT key, value FROM src WHERE key=86 ORDER BY key; DROP INDEX src_key_index ON src; -DROP INDEX src_val_index ON src; \ No newline at end of file +DROP INDEX src_val_index ON src; diff --git ql/src/test/queries/clientpositive/index_auto_partitioned.q ql/src/test/queries/clientpositive/index_auto_partitioned.q index 5013d29..70166b3 100644 --- ql/src/test/queries/clientpositive/index_auto_partitioned.q +++ ql/src/test/queries/clientpositive/index_auto_partitioned.q @@ -1,3 +1,4 @@ +set hive.stats.dbclass=fs; -- test automatic use of index on table with partitions CREATE INDEX src_part_index ON TABLE srcpart(key) as 'COMPACT' WITH DEFERRED REBUILD; ALTER INDEX src_part_index ON srcpart REBUILD; diff --git ql/src/test/queries/clientpositive/index_auto_self_join.q ql/src/test/queries/clientpositive/index_auto_self_join.q index 0984a4a..1d9efbb 100644 --- ql/src/test/queries/clientpositive/index_auto_self_join.q +++ ql/src/test/queries/clientpositive/index_auto_self_join.q @@ -3,6 +3,7 @@ EXPLAIN SELECT a.key, b.key FROM src a JOIN src b ON (a.value = b.value) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key; SELECT a.key, b.key FROM src a JOIN src b ON (a.value = b.value) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key; +set hive.stats.dbclass=fs; CREATE INDEX src_index ON TABLE src(key) as 'BITMAP' WITH DEFERRED REBUILD; ALTER INDEX src_index ON src REBUILD; diff --git ql/src/test/queries/clientpositive/index_auto_unused.q ql/src/test/queries/clientpositive/index_auto_unused.q index d8f3eda..acd4194 100644 --- ql/src/test/queries/clientpositive/index_auto_unused.q +++ ql/src/test/queries/clientpositive/index_auto_unused.q @@ -1,3 +1,4 @@ +set hive.stats.dbclass=fs; -- test cases where the index should not be used automatically CREATE INDEX src_index ON TABLE src(key) as 'COMPACT' WITH DEFERRED REBUILD; diff --git ql/src/test/queries/clientpositive/index_auto_update.q ql/src/test/queries/clientpositive/index_auto_update.q index 976de5c..c934867 100644 --- ql/src/test/queries/clientpositive/index_auto_update.q +++ ql/src/test/queries/clientpositive/index_auto_update.q @@ -1,3 +1,4 @@ +set hive.stats.dbclass=fs; -- Test if index is actually being used. -- Create temp, and populate it with some values in src. diff --git ql/src/test/queries/clientpositive/index_bitmap.q ql/src/test/queries/clientpositive/index_bitmap.q index f9deb28..673c835 100644 --- ql/src/test/queries/clientpositive/index_bitmap.q +++ ql/src/test/queries/clientpositive/index_bitmap.q @@ -1,3 +1,4 @@ +set hive.stats.dbclass=fs; DROP INDEX srcpart_index_proj on srcpart; EXPLAIN diff --git ql/src/test/queries/clientpositive/index_bitmap1.q ql/src/test/queries/clientpositive/index_bitmap1.q index 55633d9..adec8f1 100644 --- ql/src/test/queries/clientpositive/index_bitmap1.q +++ ql/src/test/queries/clientpositive/index_bitmap1.q @@ -1,3 +1,4 @@ +set hive.stats.dbclass=fs; EXPLAIN CREATE INDEX src_index ON TABLE src(key) as 'BITMAP' WITH DEFERRED REBUILD; CREATE INDEX src_index ON TABLE src(key) as 'BITMAP' WITH DEFERRED REBUILD; diff --git ql/src/test/queries/clientpositive/index_bitmap2.q ql/src/test/queries/clientpositive/index_bitmap2.q index bd15a21..1ffa6ee 100644 --- ql/src/test/queries/clientpositive/index_bitmap2.q +++ ql/src/test/queries/clientpositive/index_bitmap2.q @@ -1,3 +1,4 @@ +set hive.stats.dbclass=fs; EXPLAIN CREATE INDEX src1_index ON TABLE src(key) as 'BITMAP' WITH DEFERRED REBUILD; EXPLAIN diff --git ql/src/test/queries/clientpositive/index_bitmap_auto_partitioned.q ql/src/test/queries/clientpositive/index_bitmap_auto_partitioned.q index 90d7987..3b310ce 100644 --- ql/src/test/queries/clientpositive/index_bitmap_auto_partitioned.q +++ ql/src/test/queries/clientpositive/index_bitmap_auto_partitioned.q @@ -1,3 +1,4 @@ +set hive.stats.dbclass=fs; -- test automatic use of index on table with partitions CREATE INDEX src_part_index ON TABLE srcpart(key) as 'BITMAP' WITH DEFERRED REBUILD; ALTER INDEX src_part_index ON srcpart REBUILD; diff --git ql/src/test/queries/clientpositive/index_bitmap_compression.q ql/src/test/queries/clientpositive/index_bitmap_compression.q index 2f5e5d4..32ecfb9 100644 --- ql/src/test/queries/clientpositive/index_bitmap_compression.q +++ ql/src/test/queries/clientpositive/index_bitmap_compression.q @@ -1,3 +1,4 @@ +set hive.stats.dbclass=fs; SET hive.exec.compress.result=true; CREATE INDEX src_index ON TABLE src(key) as 'BITMAP' WITH DEFERRED REBUILD; ALTER INDEX src_index ON src REBUILD; diff --git ql/src/test/queries/clientpositive/index_bitmap_rc.q ql/src/test/queries/clientpositive/index_bitmap_rc.q index 054df51..26a351e 100644 --- ql/src/test/queries/clientpositive/index_bitmap_rc.q +++ ql/src/test/queries/clientpositive/index_bitmap_rc.q @@ -1,3 +1,4 @@ +set hive.stats.dbclass=fs; CREATE TABLE srcpart_rc (key int, value string) PARTITIONED BY (ds string, hr int) STORED AS RCFILE; INSERT OVERWRITE TABLE srcpart_rc PARTITION (ds='2008-04-08', hr=11) SELECT key, value FROM srcpart WHERE ds = '2008-04-08' AND hr = 11; diff --git ql/src/test/queries/clientpositive/index_compact.q ql/src/test/queries/clientpositive/index_compact.q index a936f11..98cbec1 100644 --- ql/src/test/queries/clientpositive/index_compact.q +++ ql/src/test/queries/clientpositive/index_compact.q @@ -1,3 +1,4 @@ +set hive.stats.dbclass=fs; DROP INDEX srcpart_index_proj on srcpart; EXPLAIN diff --git ql/src/test/queries/clientpositive/index_compact_1.q ql/src/test/queries/clientpositive/index_compact_1.q index 837033b..97276f4 100644 --- ql/src/test/queries/clientpositive/index_compact_1.q +++ ql/src/test/queries/clientpositive/index_compact_1.q @@ -1,3 +1,4 @@ +set hive.stats.dbclass=fs; EXPLAIN CREATE INDEX src_index ON TABLE src(key) as 'COMPACT' WITH DEFERRED REBUILD; CREATE INDEX src_index ON TABLE src(key) as 'COMPACT' WITH DEFERRED REBUILD; diff --git ql/src/test/queries/clientpositive/index_compact_2.q ql/src/test/queries/clientpositive/index_compact_2.q index 56119ac..1eb3f5c 100644 --- ql/src/test/queries/clientpositive/index_compact_2.q +++ ql/src/test/queries/clientpositive/index_compact_2.q @@ -1,3 +1,4 @@ +set hive.stats.dbclass=fs; CREATE TABLE srcpart_rc (key int, value string) PARTITIONED BY (ds string, hr int) STORED AS RCFILE; INSERT OVERWRITE TABLE srcpart_rc PARTITION (ds='2008-04-08', hr=11) SELECT key, value FROM srcpart WHERE ds = '2008-04-08' AND hr = 11; @@ -42,4 +43,4 @@ SET hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; SELECT key, value FROM srcpart_rc WHERE key=100 ORDER BY key; DROP INDEX srcpart_rc_index on srcpart_rc; -DROP TABLE srcpart_rc; \ No newline at end of file +DROP TABLE srcpart_rc; diff --git ql/src/test/queries/clientpositive/index_compact_3.q ql/src/test/queries/clientpositive/index_compact_3.q index f3fcb4a..599b4ac 100644 --- ql/src/test/queries/clientpositive/index_compact_3.q +++ ql/src/test/queries/clientpositive/index_compact_3.q @@ -1,3 +1,4 @@ +set hive.stats.dbclass=fs; CREATE TABLE src_index_test_rc (key int, value string) STORED AS RCFILE; INSERT OVERWRITE TABLE src_index_test_rc SELECT * FROM src; @@ -16,4 +17,4 @@ SET hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; SELECT key, value FROM src_index_test_rc WHERE key=100 ORDER BY key; DROP INDEX src_index on src_index_test_rc; -DROP TABLE src_index_test_rc; \ No newline at end of file +DROP TABLE src_index_test_rc; diff --git ql/src/test/queries/clientpositive/index_compact_binary_search.q ql/src/test/queries/clientpositive/index_compact_binary_search.q index d0d9a32..e72b27c 100644 --- ql/src/test/queries/clientpositive/index_compact_binary_search.q +++ ql/src/test/queries/clientpositive/index_compact_binary_search.q @@ -1,6 +1,6 @@ SET hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; SET hive.default.fileformat=TextFile; - +set hive.stats.dbclass=fs; CREATE INDEX src_index ON TABLE src(key) as 'COMPACT' WITH DEFERRED REBUILD; ALTER INDEX src_index ON src REBUILD; @@ -129,4 +129,4 @@ SELECT * FROM src WHERE key >= '9'; SET hive.exec.post.hooks=; -DROP INDEX src_index ON src; \ No newline at end of file +DROP INDEX src_index ON src; diff --git ql/src/test/queries/clientpositive/index_compression.q ql/src/test/queries/clientpositive/index_compression.q index 84ed3cc..963b8f7 100644 --- ql/src/test/queries/clientpositive/index_compression.q +++ ql/src/test/queries/clientpositive/index_compression.q @@ -1,4 +1,5 @@ SET hive.exec.compress.result=true; +set hive.stats.dbclass=fs; CREATE INDEX src_index ON TABLE src(key) as 'COMPACT' WITH DEFERRED REBUILD; ALTER INDEX src_index ON src REBUILD; diff --git ql/src/test/queries/clientpositive/index_creation.q ql/src/test/queries/clientpositive/index_creation.q index 062821e..ef020b6 100644 --- ql/src/test/queries/clientpositive/index_creation.q +++ ql/src/test/queries/clientpositive/index_creation.q @@ -1,3 +1,4 @@ +set hive.stats.dbclass=fs; drop index src_index_2 on src; drop index src_index_3 on src; drop index src_index_4 on src; diff --git ql/src/test/queries/clientpositive/index_serde.q ql/src/test/queries/clientpositive/index_serde.q index d160a6f..20186a7 100644 --- ql/src/test/queries/clientpositive/index_serde.q +++ ql/src/test/queries/clientpositive/index_serde.q @@ -1,3 +1,4 @@ +set hive.stats.dbclass=fs; -- Want to ensure we can build and use indices on tables stored with SerDes -- Build the (Avro backed) table CREATE TABLE doctors diff --git ql/src/test/queries/clientpositive/index_stale.q ql/src/test/queries/clientpositive/index_stale.q index 82e15b9..ecab2b7 100644 --- ql/src/test/queries/clientpositive/index_stale.q +++ ql/src/test/queries/clientpositive/index_stale.q @@ -1,3 +1,4 @@ +set hive.stats.dbclass=fs; -- test that stale indexes are not used CREATE TABLE temp(key STRING, val STRING) STORED AS TEXTFILE; diff --git ql/src/test/queries/clientpositive/index_stale_partitioned.q ql/src/test/queries/clientpositive/index_stale_partitioned.q index e7cfeff..a93ccf7 100644 --- ql/src/test/queries/clientpositive/index_stale_partitioned.q +++ ql/src/test/queries/clientpositive/index_stale_partitioned.q @@ -1,3 +1,4 @@ +set hive.stats.dbclass=fs; -- Test if index is actually being used. -- Create temp, and populate it with some values in src.