Index: eclipse-templates/.classpath._hbase =================================================================== --- eclipse-templates/.classpath._hbase (revision 0) +++ eclipse-templates/.classpath._hbase (revision 0) @@ -0,0 +1,59 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Index: eclipse-templates/TestHBaseCliDriver.launchtemplate =================================================================== --- eclipse-templates/TestHBaseCliDriver.launchtemplate (revision 0) +++ eclipse-templates/TestHBaseCliDriver.launchtemplate (revision 0) @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Index: .classpath._hbase =================================================================== --- .classpath._hbase (revision 0) +++ .classpath._hbase (revision 0) @@ -0,0 +1,59 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Index: conf/hive-default.xml =================================================================== --- conf/hive-default.xml (revision 982011) +++ conf/hive-default.xml (working copy) @@ -594,6 +594,30 @@ + hive.stats.dbclass + jdbc:derby + The default database that stores temporary hive statistics. + + + + hive.stats.autogather + true + A flag to gather statistics automatically during the INSERT OVERWRITE command. + + + + hive.stats.jdbcdriver + org.apache.derby.jdbc.EmbeddedDriver + The JDBC driver for the database that stores temporary hive statistics. + + + + hive.stats.dbconnectionstring + jdbc:derby:;databaseName=TempStatsStore;create=true + The default connection string for the database that stores temporary hive statistics. + + + fs.har.impl org.apache.hadoop.hive.shims.HiveHarFileSystem The implementation for accessing Hadoop Archives. Note that this won't be applicable to Hadoop vers less than 0.20 Index: hbase-handler/src/test/results/hbase_stats.q.out =================================================================== --- hbase-handler/src/test/results/hbase_stats.q.out (revision 0) +++ hbase-handler/src/test/results/hbase_stats.q.out (revision 0) @@ -0,0 +1,12 @@ +PREHOOK: query: analyze table srcpart compute statistics +PREHOOK: type: null +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: query: analyze table srcpart compute statistics +POSTHOOK: type: null +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 Index: hbase-handler/src/test/queries/hbase_stats.q =================================================================== --- hbase-handler/src/test/queries/hbase_stats.q (revision 0) +++ hbase-handler/src/test/queries/hbase_stats.q (revision 0) @@ -0,0 +1,2 @@ +set hive.stats.dbclass=hbase; +analyze table srcpart compute statistics; \ No newline at end of file Index: hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStatsPublisher.java =================================================================== --- hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStatsPublisher.java (revision 0) +++ hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStatsPublisher.java (revision 0) @@ -0,0 +1,152 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.hbase; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.RowLock; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hive.ql.stats.*; + +/** + * A class that implements the StatsPublisher interface through HBase. + */ +public class HBaseStatsPublisher implements StatsPublisher { + + private HTable htable; + private byte[] rowCountFamily, rowCountColumn; + private final Log LOG = LogFactory.getLog(this.getClass().getName()); + + /** + * Does the necessary HBase initializations. + */ + public boolean connect(Configuration hiveconf) { + + try { + HBaseConfiguration hbaseConf = new HBaseConfiguration(hiveconf); + HBaseAdmin hbase = new HBaseAdmin(hbaseConf); + rowCountFamily = Bytes.toBytes(HBaseStatsSetupConstants.PART_STAT_ROW_COUNT_COLUMN_FAMILY); + rowCountColumn = Bytes.toBytes(HBaseStatsSetupConstants.PART_STAT_ROW_COUNT_COLUMN_NAME); + htable = new HTable(HBaseStatsSetupConstants.PART_STAT_TABLE_NAME); + } catch (IOException e) { + LOG.error("Error during HBase initialization. " + e); + return false; + } + + return true; + } + + /** + * Writes temporary statistics into HBase; + */ + public boolean publishStat(String rowID, String key, String value) { + + boolean success = true; + + // Write in HBase + RowLock rowLock = null; + try { + rowLock = htable.lockRow(Bytes.toBytes(rowID)); + + Get get = new Get(Bytes.toBytes(rowID), rowLock); + Result result = htable.get(get); + int val = Integer.parseInt(value); + if (!result.isEmpty()) { + if (key == StatsSetupConst.ROW_COUNT) { + val += Integer.parseInt(Bytes.toString(result.getValue(rowCountFamily, rowCountColumn))); + } + else { + LOG.warn("Warning. Invalid statistic. Currently " + + "row count is the only supported statistic"); + return false; + } + } + Put row = new Put(Bytes.toBytes(rowID), rowLock); + if (key == (StatsSetupConst.ROW_COUNT)) { + row.add(rowCountFamily, rowCountColumn, Bytes.toBytes(Integer.toString(val))); + } + else { + LOG.warn("Warning. Invalid statistic. Currently " + + "row count is the only supported statistic"); + return false; + } + htable.put(row); + + } catch (IOException e) { + LOG.error("Error during publishing statistics. " + e); + success = false; + } + finally { + try { + htable.unlockRow(rowLock); + } + catch (IOException e) { + LOG.error("Error during publishing statistics. " + e); + success = false; + } + } + + return success; + } + + public boolean closeConnection() { + return true; + } + + + /** + * Does the necessary HBase initializations. + */ + public boolean init(Configuration hiveconf) { + try { + HBaseConfiguration hbaseConf = new HBaseConfiguration(hiveconf); + HBaseAdmin hbase = new HBaseAdmin(hbaseConf); + + rowCountFamily = Bytes.toBytes(HBaseStatsSetupConstants.PART_STAT_ROW_COUNT_COLUMN_FAMILY); + rowCountColumn = Bytes.toBytes(HBaseStatsSetupConstants.PART_STAT_ROW_COUNT_COLUMN_NAME); + + // Creating table if not exists + if (!hbase.tableExists(HBaseStatsSetupConstants.PART_STAT_TABLE_NAME)) { + HTableDescriptor table = new HTableDescriptor(HBaseStatsSetupConstants.PART_STAT_TABLE_NAME); + + HColumnDescriptor rowCount = new HColumnDescriptor(rowCountFamily); + table.addFamily(rowCount); + + hbase.createTable(table); + } + } catch (IOException e) { + LOG.error("Error during HBase initialization. " + e); + return false; + } + + return true; + } + +} Index: hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStatsSetupConstants.java =================================================================== --- hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStatsSetupConstants.java (revision 0) +++ hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStatsSetupConstants.java (revision 0) @@ -0,0 +1,11 @@ +package org.apache.hadoop.hive.hbase; + +public final class HBaseStatsSetupConstants { + + public static final String PART_STAT_TABLE_NAME = "PARTITION_STAT_TBL"; + + public static final String PART_STAT_ROW_COUNT_COLUMN_NAME = "ROW_COUNT"; + + public static final String PART_STAT_ROW_COUNT_COLUMN_FAMILY = "ROW_COUNT_FAMILY"; + +} Index: hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStatsAggregator.java =================================================================== --- hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStatsAggregator.java (revision 0) +++ hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStatsAggregator.java (revision 0) @@ -0,0 +1,110 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.hbase; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Iterator; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hive.ql.stats.*; + + +/** + * A class that implements the StatsAggregator interface through HBase. + */ +public class HBaseStatsAggregator implements StatsAggregator { + + private HTable htable; + private byte[] rowCountFamily, rowCountColumn; + private final Log LOG = LogFactory.getLog(this.getClass().getName()); + + /** + * Does the necessary HBase initializations. + */ + public boolean connect(Configuration hiveconf) { + + try { + HBaseConfiguration hbaseConf = new HBaseConfiguration(hiveconf); + HBaseAdmin hbase = new HBaseAdmin(hbaseConf); + rowCountFamily = Bytes.toBytes(HBaseStatsSetupConstants.PART_STAT_ROW_COUNT_COLUMN_FAMILY); + rowCountColumn = Bytes.toBytes(HBaseStatsSetupConstants.PART_STAT_ROW_COUNT_COLUMN_NAME); + htable = new HTable(HBaseStatsSetupConstants.PART_STAT_TABLE_NAME); + + } catch (IOException e) { + LOG.error("Error during HBase initializations. " + e); + return false; + } + + return true; + } + + /** + * Aggregates temporary stats from HBase; + */ + public String aggregateStats(String rowID, String key) { + + byte[] retValue = null; + + try { + Get get = new Get(Bytes.toBytes(rowID)); + Result result = htable.get(get); + + // Row Count + if (key == StatsSetupConst.ROW_COUNT) { + retValue = result.getValue(rowCountFamily, rowCountColumn); + /* Automatic Cleaning: + IMPORTANT: Since we publish and aggregate only 1 value (1 column) which is the row count, it + is valid to delete the row after aggregation (automatic cleaning) because we know that there is no + other values to aggregate. + If ;in the future; other values are aggregated and published, then we cannot do cleaning except + when we are sure that all values are aggregated, or we can separate the implementation of cleaning + through a separate method which the developer has to call it manually in the code. + */ + Delete delete = new Delete(Bytes.toBytes(rowID)); + htable.delete(delete); + } + else { + LOG.warn("Warning. Invalid statistic. Currently " + + "row count is the only supported statistic"); + return null; + } + } catch (IOException e) { + LOG.error("Error during publishing aggregation. " + e); + return null; + } + + return Bytes.toString(retValue); + } + + public boolean closeConnection() { + return true; + } + +} Index: lib/mysql-connector-java-5.1.6-bin.jar =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: lib/mysql-connector-java-5.1.6-bin.jar ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: build-common.xml =================================================================== --- build-common.xml (revision 982011) +++ build-common.xml (working copy) @@ -412,7 +412,7 @@ --> - + Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 982011) +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -262,6 +262,12 @@ HIVEOPTSORTMERGEBUCKETMAPJOIN("hive.optimize.bucketmapjoin.sortedmerge", false), // try to use sorted merge bucket map join HIVEOPTREDUCEDEDUPLICATION("hive.optimize.reducededuplication", true), + // Statistics + HIVESTATSDBCLASS("hive.stats.dbclass", "jdbc:derby"), // jdbc:derby, jdbc:mysql, hbase or whatever. See StatsSetupConst.java to set up these values + HIVESTATSAUTOGATHER("hive.stats.autogather", true), // To gather statistics automatically during the INSERT OVERWRITE command + HIVESTATSJDBCDRIVER("hive.stats.jdbcdriver", "org.apache.derby.jdbc.EmbeddedDriver"), + HIVESTATSDBCONNECTIONSTRING("hive.stats.dbconnectionstring", "jdbc:derby:;databaseName=TempStatsStore;create=true"),//jdbc:mysql://cdb043.sf2p.facebook.com/stats_ahmed_test;user=www;password=A3V6IPS1vg + // For HBase storage handler HIVE_HBASE_WAL_ENABLED("hive.hbase.wal.enabled", true), Index: ql/src/test/results/clientpositive/stats8.q.out =================================================================== --- ql/src/test/results/clientpositive/stats8.q.out (revision 0) +++ ql/src/test/results/clientpositive/stats8.q.out (revision 0) @@ -0,0 +1,52 @@ +PREHOOK: query: analyze table srcpart PARTITION(ds, hr) compute statistics +PREHOOK: type: null +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: query: analyze table srcpart PARTITION(ds, hr) compute statistics +POSTHOOK: type: null +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: query: describe extended srcpart PARTITION(ds='2008-04-08',hr=11) +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe extended srcpart PARTITION(ds='2008-04-08',hr=11) +POSTHOOK: type: DESCTABLE +key string default +value string default +ds string +hr string + +Detailed Partition Information Partition(values:[2008-04-08, 11], dbName:default, tableName:srcpart, createTime:1280872528, lastAccessTime:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:default), FieldSchema(name:value, type:string, comment:default)], location:pfile:/data/users/aaly/work/apache-hive/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), parameters:{numFiles=1, transient_lastDdlTime=1280877203, numRows=500, totalSize=4096}) +PREHOOK: query: describe extended srcpart PARTITION(ds='2008-04-09',hr=12) +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe extended srcpart PARTITION(ds='2008-04-09',hr=12) +POSTHOOK: type: DESCTABLE +key string default +value string default +ds string +hr string + +Detailed Partition Information Partition(values:[2008-04-09, 12], dbName:default, tableName:srcpart, createTime:1280872529, lastAccessTime:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:default), FieldSchema(name:value, type:string, comment:default)], location:pfile:/data/users/aaly/work/apache-hive/build/ql/test/data/warehouse/srcpart/ds=2008-04-09/hr=12, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), parameters:{numFiles=1, transient_lastDdlTime=1280877203, numRows=500, totalSize=4096}) +PREHOOK: query: describe extended srcpart PARTITION(ds='2008-04-08',hr=11) +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe extended srcpart PARTITION(ds='2008-04-08',hr=11) +POSTHOOK: type: DESCTABLE +key string default +value string default +ds string +hr string + +Detailed Partition Information Partition(values:[2008-04-08, 11], dbName:default, tableName:srcpart, createTime:1280872528, lastAccessTime:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:default), FieldSchema(name:value, type:string, comment:default)], location:pfile:/data/users/aaly/work/apache-hive/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), parameters:{numFiles=1, transient_lastDdlTime=1280877203, numRows=500, totalSize=4096}) +PREHOOK: query: describe extended srcpart PARTITION(ds='2008-04-09',hr=12) +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe extended srcpart PARTITION(ds='2008-04-09',hr=12) +POSTHOOK: type: DESCTABLE +key string default +value string default +ds string +hr string + +Detailed Partition Information Partition(values:[2008-04-09, 12], dbName:default, tableName:srcpart, createTime:1280872529, lastAccessTime:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:default), FieldSchema(name:value, type:string, comment:default)], location:pfile:/data/users/aaly/work/apache-hive/build/ql/test/data/warehouse/srcpart/ds=2008-04-09/hr=12, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), parameters:{numFiles=1, transient_lastDdlTime=1280877203, numRows=500, totalSize=4096}) Index: ql/src/test/results/clientpositive/stats3.q.out =================================================================== --- ql/src/test/results/clientpositive/stats3.q.out (revision 0) +++ ql/src/test/results/clientpositive/stats3.q.out (revision 0) @@ -0,0 +1,132 @@ +PREHOOK: query: drop table hive_test_src +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table hive_test_src +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table hive_test_dst +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table hive_test_dst +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table hive_test_src ( col1 string ) stored as textfile +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table hive_test_src ( col1 string ) stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@hive_test_src +PREHOOK: query: load data local inpath '../data/files/test.dat' overwrite into table hive_test_src +PREHOOK: type: LOAD +POSTHOOK: query: load data local inpath '../data/files/test.dat' overwrite into table hive_test_src +POSTHOOK: type: LOAD +POSTHOOK: Output: default@hive_test_src +PREHOOK: query: create table hive_test_dst ( col1 string ) partitioned by ( pcol1 string , pcol2 string) stored as sequencefile +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table hive_test_dst ( col1 string ) partitioned by ( pcol1 string , pcol2 string) stored as sequencefile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@hive_test_dst +PREHOOK: query: insert overwrite table hive_test_dst partition ( pcol1='test_part', pCol2='test_Part') select col1 from hive_test_src +PREHOOK: type: QUERY +PREHOOK: Input: default@hive_test_src +PREHOOK: Output: default@hive_test_dst@pcol1=test_part/pcol2=test_Part +POSTHOOK: query: insert overwrite table hive_test_dst partition ( pcol1='test_part', pCol2='test_Part') select col1 from hive_test_src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hive_test_src +POSTHOOK: Output: default@hive_test_dst@pcol1=test_part/pcol2=test_Part +POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] +PREHOOK: query: select * from hive_test_dst where pcol1='test_part' and pcol2='test_Part' +PREHOOK: type: QUERY +PREHOOK: Input: default@hive_test_dst@pcol1=test_part/pcol2=test_Part +PREHOOK: Output: file:/tmp/aaly/hive_2010-08-04_12-48-16_367_3909580618833730976/-mr-10000 +POSTHOOK: query: select * from hive_test_dst where pcol1='test_part' and pcol2='test_Part' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hive_test_dst@pcol1=test_part/pcol2=test_Part +POSTHOOK: Output: file:/tmp/aaly/hive_2010-08-04_12-48-16_367_3909580618833730976/-mr-10000 +POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] +1 test_part test_Part +2 test_part test_Part +3 test_part test_Part +4 test_part test_Part +5 test_part test_Part +6 test_part test_Part +PREHOOK: query: select count(1) from hive_test_dst +PREHOOK: type: QUERY +PREHOOK: Input: default@hive_test_dst@pcol1=test_part/pcol2=test_Part +PREHOOK: Output: file:/tmp/aaly/hive_2010-08-04_12-48-16_683_1151783909114489660/-mr-10000 +POSTHOOK: query: select count(1) from hive_test_dst +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hive_test_dst@pcol1=test_part/pcol2=test_Part +POSTHOOK: Output: file:/tmp/aaly/hive_2010-08-04_12-48-16_683_1151783909114489660/-mr-10000 +POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] +6 +PREHOOK: query: insert overwrite table hive_test_dst partition ( pCol1='test_part', pcol2='test_Part') select col1 from hive_test_src +PREHOOK: type: QUERY +PREHOOK: Input: default@hive_test_src +PREHOOK: Output: default@hive_test_dst@pcol1=test_part/pcol2=test_Part +POSTHOOK: query: insert overwrite table hive_test_dst partition ( pCol1='test_part', pcol2='test_Part') select col1 from hive_test_src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hive_test_src +POSTHOOK: Output: default@hive_test_dst@pcol1=test_part/pcol2=test_Part +POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] +POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] +PREHOOK: query: select * from hive_test_dst where pcol1='test_part' and pcol2='test_part' +PREHOOK: type: QUERY +PREHOOK: Output: file:/tmp/aaly/hive_2010-08-04_12-48-23_249_3869046444303707171/-mr-10000 +POSTHOOK: query: select * from hive_test_dst where pcol1='test_part' and pcol2='test_part' +POSTHOOK: type: QUERY +POSTHOOK: Output: file:/tmp/aaly/hive_2010-08-04_12-48-23_249_3869046444303707171/-mr-10000 +POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] +POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] +PREHOOK: query: select count(1) from hive_test_dst +PREHOOK: type: QUERY +PREHOOK: Input: default@hive_test_dst@pcol1=test_part/pcol2=test_Part +PREHOOK: Output: file:/tmp/aaly/hive_2010-08-04_12-48-23_307_7755470075971345346/-mr-10000 +POSTHOOK: query: select count(1) from hive_test_dst +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hive_test_dst@pcol1=test_part/pcol2=test_Part +POSTHOOK: Output: file:/tmp/aaly/hive_2010-08-04_12-48-23_307_7755470075971345346/-mr-10000 +POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] +POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] +6 +PREHOOK: query: select * from hive_test_dst where pcol1='test_part' +PREHOOK: type: QUERY +PREHOOK: Input: default@hive_test_dst@pcol1=test_part/pcol2=test_Part +PREHOOK: Output: file:/tmp/aaly/hive_2010-08-04_12-48-25_902_4804840223806837862/-mr-10000 +POSTHOOK: query: select * from hive_test_dst where pcol1='test_part' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hive_test_dst@pcol1=test_part/pcol2=test_Part +POSTHOOK: Output: file:/tmp/aaly/hive_2010-08-04_12-48-25_902_4804840223806837862/-mr-10000 +POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] +POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] +1 test_part test_Part +2 test_part test_Part +3 test_part test_Part +4 test_part test_Part +5 test_part test_Part +6 test_part test_Part +PREHOOK: query: select * from hive_test_dst where pcol1='test_part' and pcol2='test_part' +PREHOOK: type: QUERY +PREHOOK: Output: file:/tmp/aaly/hive_2010-08-04_12-48-26_132_1036351978829772706/-mr-10000 +POSTHOOK: query: select * from hive_test_dst where pcol1='test_part' and pcol2='test_part' +POSTHOOK: type: QUERY +POSTHOOK: Output: file:/tmp/aaly/hive_2010-08-04_12-48-26_132_1036351978829772706/-mr-10000 +POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] +POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] +PREHOOK: query: select * from hive_test_dst where pcol1='test_Part' +PREHOOK: type: QUERY +PREHOOK: Output: file:/tmp/aaly/hive_2010-08-04_12-48-26_185_545787673493639708/-mr-10000 +POSTHOOK: query: select * from hive_test_dst where pcol1='test_Part' +POSTHOOK: type: QUERY +POSTHOOK: Output: file:/tmp/aaly/hive_2010-08-04_12-48-26_185_545787673493639708/-mr-10000 +POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] +POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] +PREHOOK: query: drop table hive_test_src +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table hive_test_src +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: default@hive_test_src +POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] +POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] +PREHOOK: query: drop table hive_test_dst +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table hive_test_dst +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: default@hive_test_dst +POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] +POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] Index: ql/src/test/results/clientpositive/stats9.q.out =================================================================== --- ql/src/test/results/clientpositive/stats9.q.out (revision 0) +++ ql/src/test/results/clientpositive/stats9.q.out (revision 0) @@ -0,0 +1,28 @@ +PREHOOK: query: analyze table srcbucket compute statistics +PREHOOK: type: null +PREHOOK: Input: default@srcbucket +POSTHOOK: query: analyze table srcbucket compute statistics +POSTHOOK: type: null +POSTHOOK: Input: default@srcbucket +PREHOOK: query: describe extended srcbucket +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe extended srcbucket +POSTHOOK: type: DESCTABLE +key int +value string + +Detailed Table Information Table(tableName:srcbucket, dbName:default, owner:aaly, createTime:1280776830, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:int, comment:null), FieldSchema(name:value, type:string, comment:null)], location:pfile:/data/users/aaly/work/apache-hive/build/ql/test/data/warehouse/srcbucket, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:2, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[key], sortCols:[], parameters:{}), partitionKeys:[], parameters:{numPartitions=0, numFiles=1, transient_lastDdlTime=1280776838, numRows=1000, totalSize=4096}, viewOriginalText:null, viewExpandedText:null, tableType:MANAGED_TABLE) +PREHOOK: query: analyze table srcbucket2 compute statistics +PREHOOK: type: null +PREHOOK: Input: default@srcbucket2 +POSTHOOK: query: analyze table srcbucket2 compute statistics +POSTHOOK: type: null +POSTHOOK: Input: default@srcbucket2 +PREHOOK: query: describe extended srcbucket2 +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe extended srcbucket2 +POSTHOOK: type: DESCTABLE +key int +value string + +Detailed Table Information Table(tableName:srcbucket2, dbName:default, owner:aaly, createTime:1280776831, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:int, comment:null), FieldSchema(name:value, type:string, comment:null)], location:pfile:/data/users/aaly/work/apache-hive/build/ql/test/data/warehouse/srcbucket2, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:4, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[key], sortCols:[], parameters:{}), partitionKeys:[], parameters:{numPartitions=0, numFiles=1, transient_lastDdlTime=1280776842, numRows=500, totalSize=4096}, viewOriginalText:null, viewExpandedText:null, tableType:MANAGED_TABLE) Index: ql/src/test/results/clientpositive/stats4.q.out =================================================================== --- ql/src/test/results/clientpositive/stats4.q.out (revision 0) +++ ql/src/test/results/clientpositive/stats4.q.out (revision 0) @@ -0,0 +1,2357 @@ +PREHOOK: query: show partitions srcpart +PREHOOK: type: SHOWPARTITIONS +POSTHOOK: query: show partitions srcpart +POSTHOOK: type: SHOWPARTITIONS +ds=2008-04-08/hr=11 +ds=2008-04-08/hr=12 +ds=2008-04-09/hr=11 +ds=2008-04-09/hr=12 +PREHOOK: query: drop table nzhang_part1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table nzhang_part1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table nzhang_part2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table nzhang_part2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table if not exists nzhang_part1 like srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table if not exists nzhang_part1 like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@nzhang_part1 +PREHOOK: query: create table if not exists nzhang_part2 like srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table if not exists nzhang_part2 like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@nzhang_part2 +PREHOOK: query: explain +from srcpart +insert overwrite table nzhang_part1 partition (ds, hr) select key, value, ds, hr where ds <= '2008-04-08' +insert overwrite table nzhang_part2 partition(ds='2008-12-31', hr) select key, value, hr where ds > '2008-04-08' +PREHOOK: type: QUERY +POSTHOOK: query: explain +from srcpart +insert overwrite table nzhang_part1 partition (ds, hr) select key, value, ds, hr where ds <= '2008-04-08' +insert overwrite table nzhang_part2 partition(ds='2008-12-31', hr) select key, value, hr where ds > '2008-04-08' +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF srcpart)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB nzhang_part1 (TOK_PARTSPEC (TOK_PARTVAL ds) (TOK_PARTVAL hr)))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_TABLE_OR_COL ds)) (TOK_SELEXPR (TOK_TABLE_OR_COL hr))) (TOK_WHERE (<= (TOK_TABLE_OR_COL ds) '2008-04-08'))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB nzhang_part2 (TOK_PARTSPEC (TOK_PARTVAL ds '2008-12-31') (TOK_PARTVAL hr)))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_TABLE_OR_COL hr))) (TOK_WHERE (> (TOK_TABLE_OR_COL ds) '2008-04-08')))) + +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-2 + Stage-4 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + srcpart + TableScan + alias: srcpart + Filter Operator + predicate: + expr: (ds <= '2008-04-08') + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + expr: ds + type: string + expr: hr + type: string + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: nzhang_part1 + Filter Operator + predicate: + expr: (ds > '2008-04-08') + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + expr: hr + type: string + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 2 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: nzhang_part2 + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + hr + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: nzhang_part1 + + Stage: Stage-3 + Stats Operator + + Stage: Stage-1 + Move Operator + tables: + partition: + ds 2008-12-31 + hr + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: nzhang_part2 + + Stage: Stage-4 + Stats Operator + + +PREHOOK: query: from srcpart +insert overwrite table nzhang_part1 partition (ds, hr) select key, value, ds, hr where ds <= '2008-04-08' +insert overwrite table nzhang_part2 partition(ds='2008-12-31', hr) select key, value, hr where ds > '2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: query: from srcpart +insert overwrite table nzhang_part1 partition (ds, hr) select key, value, ds, hr where ds <= '2008-04-08' +insert overwrite table nzhang_part2 partition(ds='2008-12-31', hr) select key, value, hr where ds > '2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@nzhang_part1@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@nzhang_part1@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@nzhang_part2@ds=2008-12-31/hr=11 +POSTHOOK: Output: default@nzhang_part2@ds=2008-12-31/hr=12 +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +PREHOOK: query: show partitions nzhang_part1 +PREHOOK: type: SHOWPARTITIONS +POSTHOOK: query: show partitions nzhang_part1 +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +ds=2008-04-08/hr=11 +ds=2008-04-08/hr=12 +PREHOOK: query: show partitions nzhang_part2 +PREHOOK: type: SHOWPARTITIONS +POSTHOOK: query: show partitions nzhang_part2 +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +ds=2008-12-31/hr=11 +ds=2008-12-31/hr=12 +PREHOOK: query: select * from nzhang_part1 where ds is not null and hr is not null +PREHOOK: type: QUERY +PREHOOK: Input: default@nzhang_part1@ds=2008-04-08/hr=11 +PREHOOK: Input: default@nzhang_part1@ds=2008-04-08/hr=12 +PREHOOK: Output: file:/tmp/aaly/hive_2010-08-09_13-10-49_944_7480353582912969356/-mr-10000 +POSTHOOK: query: select * from nzhang_part1 where ds is not null and hr is not null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@nzhang_part1@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@nzhang_part1@ds=2008-04-08/hr=12 +POSTHOOK: Output: file:/tmp/aaly/hive_2010-08-09_13-10-49_944_7480353582912969356/-mr-10000 +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +238 val_238 2008-04-08 11 +86 val_86 2008-04-08 11 +311 val_311 2008-04-08 11 +27 val_27 2008-04-08 11 +165 val_165 2008-04-08 11 +409 val_409 2008-04-08 11 +255 val_255 2008-04-08 11 +278 val_278 2008-04-08 11 +98 val_98 2008-04-08 11 +484 val_484 2008-04-08 11 +265 val_265 2008-04-08 11 +193 val_193 2008-04-08 11 +401 val_401 2008-04-08 11 +150 val_150 2008-04-08 11 +273 val_273 2008-04-08 11 +224 val_224 2008-04-08 11 +369 val_369 2008-04-08 11 +66 val_66 2008-04-08 11 +128 val_128 2008-04-08 11 +213 val_213 2008-04-08 11 +146 val_146 2008-04-08 11 +406 val_406 2008-04-08 11 +429 val_429 2008-04-08 11 +374 val_374 2008-04-08 11 +152 val_152 2008-04-08 11 +469 val_469 2008-04-08 11 +145 val_145 2008-04-08 11 +495 val_495 2008-04-08 11 +37 val_37 2008-04-08 11 +327 val_327 2008-04-08 11 +281 val_281 2008-04-08 11 +277 val_277 2008-04-08 11 +209 val_209 2008-04-08 11 +15 val_15 2008-04-08 11 +82 val_82 2008-04-08 11 +403 val_403 2008-04-08 11 +166 val_166 2008-04-08 11 +417 val_417 2008-04-08 11 +430 val_430 2008-04-08 11 +252 val_252 2008-04-08 11 +292 val_292 2008-04-08 11 +219 val_219 2008-04-08 11 +287 val_287 2008-04-08 11 +153 val_153 2008-04-08 11 +193 val_193 2008-04-08 11 +338 val_338 2008-04-08 11 +446 val_446 2008-04-08 11 +459 val_459 2008-04-08 11 +394 val_394 2008-04-08 11 +237 val_237 2008-04-08 11 +482 val_482 2008-04-08 11 +174 val_174 2008-04-08 11 +413 val_413 2008-04-08 11 +494 val_494 2008-04-08 11 +207 val_207 2008-04-08 11 +199 val_199 2008-04-08 11 +466 val_466 2008-04-08 11 +208 val_208 2008-04-08 11 +174 val_174 2008-04-08 11 +399 val_399 2008-04-08 11 +396 val_396 2008-04-08 11 +247 val_247 2008-04-08 11 +417 val_417 2008-04-08 11 +489 val_489 2008-04-08 11 +162 val_162 2008-04-08 11 +377 val_377 2008-04-08 11 +397 val_397 2008-04-08 11 +309 val_309 2008-04-08 11 +365 val_365 2008-04-08 11 +266 val_266 2008-04-08 11 +439 val_439 2008-04-08 11 +342 val_342 2008-04-08 11 +367 val_367 2008-04-08 11 +325 val_325 2008-04-08 11 +167 val_167 2008-04-08 11 +195 val_195 2008-04-08 11 +475 val_475 2008-04-08 11 +17 val_17 2008-04-08 11 +113 val_113 2008-04-08 11 +155 val_155 2008-04-08 11 +203 val_203 2008-04-08 11 +339 val_339 2008-04-08 11 +0 val_0 2008-04-08 11 +455 val_455 2008-04-08 11 +128 val_128 2008-04-08 11 +311 val_311 2008-04-08 11 +316 val_316 2008-04-08 11 +57 val_57 2008-04-08 11 +302 val_302 2008-04-08 11 +205 val_205 2008-04-08 11 +149 val_149 2008-04-08 11 +438 val_438 2008-04-08 11 +345 val_345 2008-04-08 11 +129 val_129 2008-04-08 11 +170 val_170 2008-04-08 11 +20 val_20 2008-04-08 11 +489 val_489 2008-04-08 11 +157 val_157 2008-04-08 11 +378 val_378 2008-04-08 11 +221 val_221 2008-04-08 11 +92 val_92 2008-04-08 11 +111 val_111 2008-04-08 11 +47 val_47 2008-04-08 11 +72 val_72 2008-04-08 11 +4 val_4 2008-04-08 11 +280 val_280 2008-04-08 11 +35 val_35 2008-04-08 11 +427 val_427 2008-04-08 11 +277 val_277 2008-04-08 11 +208 val_208 2008-04-08 11 +356 val_356 2008-04-08 11 +399 val_399 2008-04-08 11 +169 val_169 2008-04-08 11 +382 val_382 2008-04-08 11 +498 val_498 2008-04-08 11 +125 val_125 2008-04-08 11 +386 val_386 2008-04-08 11 +437 val_437 2008-04-08 11 +469 val_469 2008-04-08 11 +192 val_192 2008-04-08 11 +286 val_286 2008-04-08 11 +187 val_187 2008-04-08 11 +176 val_176 2008-04-08 11 +54 val_54 2008-04-08 11 +459 val_459 2008-04-08 11 +51 val_51 2008-04-08 11 +138 val_138 2008-04-08 11 +103 val_103 2008-04-08 11 +239 val_239 2008-04-08 11 +213 val_213 2008-04-08 11 +216 val_216 2008-04-08 11 +430 val_430 2008-04-08 11 +278 val_278 2008-04-08 11 +176 val_176 2008-04-08 11 +289 val_289 2008-04-08 11 +221 val_221 2008-04-08 11 +65 val_65 2008-04-08 11 +318 val_318 2008-04-08 11 +332 val_332 2008-04-08 11 +311 val_311 2008-04-08 11 +275 val_275 2008-04-08 11 +137 val_137 2008-04-08 11 +241 val_241 2008-04-08 11 +83 val_83 2008-04-08 11 +333 val_333 2008-04-08 11 +180 val_180 2008-04-08 11 +284 val_284 2008-04-08 11 +12 val_12 2008-04-08 11 +230 val_230 2008-04-08 11 +181 val_181 2008-04-08 11 +67 val_67 2008-04-08 11 +260 val_260 2008-04-08 11 +404 val_404 2008-04-08 11 +384 val_384 2008-04-08 11 +489 val_489 2008-04-08 11 +353 val_353 2008-04-08 11 +373 val_373 2008-04-08 11 +272 val_272 2008-04-08 11 +138 val_138 2008-04-08 11 +217 val_217 2008-04-08 11 +84 val_84 2008-04-08 11 +348 val_348 2008-04-08 11 +466 val_466 2008-04-08 11 +58 val_58 2008-04-08 11 +8 val_8 2008-04-08 11 +411 val_411 2008-04-08 11 +230 val_230 2008-04-08 11 +208 val_208 2008-04-08 11 +348 val_348 2008-04-08 11 +24 val_24 2008-04-08 11 +463 val_463 2008-04-08 11 +431 val_431 2008-04-08 11 +179 val_179 2008-04-08 11 +172 val_172 2008-04-08 11 +42 val_42 2008-04-08 11 +129 val_129 2008-04-08 11 +158 val_158 2008-04-08 11 +119 val_119 2008-04-08 11 +496 val_496 2008-04-08 11 +0 val_0 2008-04-08 11 +322 val_322 2008-04-08 11 +197 val_197 2008-04-08 11 +468 val_468 2008-04-08 11 +393 val_393 2008-04-08 11 +454 val_454 2008-04-08 11 +100 val_100 2008-04-08 11 +298 val_298 2008-04-08 11 +199 val_199 2008-04-08 11 +191 val_191 2008-04-08 11 +418 val_418 2008-04-08 11 +96 val_96 2008-04-08 11 +26 val_26 2008-04-08 11 +165 val_165 2008-04-08 11 +327 val_327 2008-04-08 11 +230 val_230 2008-04-08 11 +205 val_205 2008-04-08 11 +120 val_120 2008-04-08 11 +131 val_131 2008-04-08 11 +51 val_51 2008-04-08 11 +404 val_404 2008-04-08 11 +43 val_43 2008-04-08 11 +436 val_436 2008-04-08 11 +156 val_156 2008-04-08 11 +469 val_469 2008-04-08 11 +468 val_468 2008-04-08 11 +308 val_308 2008-04-08 11 +95 val_95 2008-04-08 11 +196 val_196 2008-04-08 11 +288 val_288 2008-04-08 11 +481 val_481 2008-04-08 11 +457 val_457 2008-04-08 11 +98 val_98 2008-04-08 11 +282 val_282 2008-04-08 11 +197 val_197 2008-04-08 11 +187 val_187 2008-04-08 11 +318 val_318 2008-04-08 11 +318 val_318 2008-04-08 11 +409 val_409 2008-04-08 11 +470 val_470 2008-04-08 11 +137 val_137 2008-04-08 11 +369 val_369 2008-04-08 11 +316 val_316 2008-04-08 11 +169 val_169 2008-04-08 11 +413 val_413 2008-04-08 11 +85 val_85 2008-04-08 11 +77 val_77 2008-04-08 11 +0 val_0 2008-04-08 11 +490 val_490 2008-04-08 11 +87 val_87 2008-04-08 11 +364 val_364 2008-04-08 11 +179 val_179 2008-04-08 11 +118 val_118 2008-04-08 11 +134 val_134 2008-04-08 11 +395 val_395 2008-04-08 11 +282 val_282 2008-04-08 11 +138 val_138 2008-04-08 11 +238 val_238 2008-04-08 11 +419 val_419 2008-04-08 11 +15 val_15 2008-04-08 11 +118 val_118 2008-04-08 11 +72 val_72 2008-04-08 11 +90 val_90 2008-04-08 11 +307 val_307 2008-04-08 11 +19 val_19 2008-04-08 11 +435 val_435 2008-04-08 11 +10 val_10 2008-04-08 11 +277 val_277 2008-04-08 11 +273 val_273 2008-04-08 11 +306 val_306 2008-04-08 11 +224 val_224 2008-04-08 11 +309 val_309 2008-04-08 11 +389 val_389 2008-04-08 11 +327 val_327 2008-04-08 11 +242 val_242 2008-04-08 11 +369 val_369 2008-04-08 11 +392 val_392 2008-04-08 11 +272 val_272 2008-04-08 11 +331 val_331 2008-04-08 11 +401 val_401 2008-04-08 11 +242 val_242 2008-04-08 11 +452 val_452 2008-04-08 11 +177 val_177 2008-04-08 11 +226 val_226 2008-04-08 11 +5 val_5 2008-04-08 11 +497 val_497 2008-04-08 11 +402 val_402 2008-04-08 11 +396 val_396 2008-04-08 11 +317 val_317 2008-04-08 11 +395 val_395 2008-04-08 11 +58 val_58 2008-04-08 11 +35 val_35 2008-04-08 11 +336 val_336 2008-04-08 11 +95 val_95 2008-04-08 11 +11 val_11 2008-04-08 11 +168 val_168 2008-04-08 11 +34 val_34 2008-04-08 11 +229 val_229 2008-04-08 11 +233 val_233 2008-04-08 11 +143 val_143 2008-04-08 11 +472 val_472 2008-04-08 11 +322 val_322 2008-04-08 11 +498 val_498 2008-04-08 11 +160 val_160 2008-04-08 11 +195 val_195 2008-04-08 11 +42 val_42 2008-04-08 11 +321 val_321 2008-04-08 11 +430 val_430 2008-04-08 11 +119 val_119 2008-04-08 11 +489 val_489 2008-04-08 11 +458 val_458 2008-04-08 11 +78 val_78 2008-04-08 11 +76 val_76 2008-04-08 11 +41 val_41 2008-04-08 11 +223 val_223 2008-04-08 11 +492 val_492 2008-04-08 11 +149 val_149 2008-04-08 11 +449 val_449 2008-04-08 11 +218 val_218 2008-04-08 11 +228 val_228 2008-04-08 11 +138 val_138 2008-04-08 11 +453 val_453 2008-04-08 11 +30 val_30 2008-04-08 11 +209 val_209 2008-04-08 11 +64 val_64 2008-04-08 11 +468 val_468 2008-04-08 11 +76 val_76 2008-04-08 11 +74 val_74 2008-04-08 11 +342 val_342 2008-04-08 11 +69 val_69 2008-04-08 11 +230 val_230 2008-04-08 11 +33 val_33 2008-04-08 11 +368 val_368 2008-04-08 11 +103 val_103 2008-04-08 11 +296 val_296 2008-04-08 11 +113 val_113 2008-04-08 11 +216 val_216 2008-04-08 11 +367 val_367 2008-04-08 11 +344 val_344 2008-04-08 11 +167 val_167 2008-04-08 11 +274 val_274 2008-04-08 11 +219 val_219 2008-04-08 11 +239 val_239 2008-04-08 11 +485 val_485 2008-04-08 11 +116 val_116 2008-04-08 11 +223 val_223 2008-04-08 11 +256 val_256 2008-04-08 11 +263 val_263 2008-04-08 11 +70 val_70 2008-04-08 11 +487 val_487 2008-04-08 11 +480 val_480 2008-04-08 11 +401 val_401 2008-04-08 11 +288 val_288 2008-04-08 11 +191 val_191 2008-04-08 11 +5 val_5 2008-04-08 11 +244 val_244 2008-04-08 11 +438 val_438 2008-04-08 11 +128 val_128 2008-04-08 11 +467 val_467 2008-04-08 11 +432 val_432 2008-04-08 11 +202 val_202 2008-04-08 11 +316 val_316 2008-04-08 11 +229 val_229 2008-04-08 11 +469 val_469 2008-04-08 11 +463 val_463 2008-04-08 11 +280 val_280 2008-04-08 11 +2 val_2 2008-04-08 11 +35 val_35 2008-04-08 11 +283 val_283 2008-04-08 11 +331 val_331 2008-04-08 11 +235 val_235 2008-04-08 11 +80 val_80 2008-04-08 11 +44 val_44 2008-04-08 11 +193 val_193 2008-04-08 11 +321 val_321 2008-04-08 11 +335 val_335 2008-04-08 11 +104 val_104 2008-04-08 11 +466 val_466 2008-04-08 11 +366 val_366 2008-04-08 11 +175 val_175 2008-04-08 11 +403 val_403 2008-04-08 11 +483 val_483 2008-04-08 11 +53 val_53 2008-04-08 11 +105 val_105 2008-04-08 11 +257 val_257 2008-04-08 11 +406 val_406 2008-04-08 11 +409 val_409 2008-04-08 11 +190 val_190 2008-04-08 11 +406 val_406 2008-04-08 11 +401 val_401 2008-04-08 11 +114 val_114 2008-04-08 11 +258 val_258 2008-04-08 11 +90 val_90 2008-04-08 11 +203 val_203 2008-04-08 11 +262 val_262 2008-04-08 11 +348 val_348 2008-04-08 11 +424 val_424 2008-04-08 11 +12 val_12 2008-04-08 11 +396 val_396 2008-04-08 11 +201 val_201 2008-04-08 11 +217 val_217 2008-04-08 11 +164 val_164 2008-04-08 11 +431 val_431 2008-04-08 11 +454 val_454 2008-04-08 11 +478 val_478 2008-04-08 11 +298 val_298 2008-04-08 11 +125 val_125 2008-04-08 11 +431 val_431 2008-04-08 11 +164 val_164 2008-04-08 11 +424 val_424 2008-04-08 11 +187 val_187 2008-04-08 11 +382 val_382 2008-04-08 11 +5 val_5 2008-04-08 11 +70 val_70 2008-04-08 11 +397 val_397 2008-04-08 11 +480 val_480 2008-04-08 11 +291 val_291 2008-04-08 11 +24 val_24 2008-04-08 11 +351 val_351 2008-04-08 11 +255 val_255 2008-04-08 11 +104 val_104 2008-04-08 11 +70 val_70 2008-04-08 11 +163 val_163 2008-04-08 11 +438 val_438 2008-04-08 11 +119 val_119 2008-04-08 11 +414 val_414 2008-04-08 11 +200 val_200 2008-04-08 11 +491 val_491 2008-04-08 11 +237 val_237 2008-04-08 11 +439 val_439 2008-04-08 11 +360 val_360 2008-04-08 11 +248 val_248 2008-04-08 11 +479 val_479 2008-04-08 11 +305 val_305 2008-04-08 11 +417 val_417 2008-04-08 11 +199 val_199 2008-04-08 11 +444 val_444 2008-04-08 11 +120 val_120 2008-04-08 11 +429 val_429 2008-04-08 11 +169 val_169 2008-04-08 11 +443 val_443 2008-04-08 11 +323 val_323 2008-04-08 11 +325 val_325 2008-04-08 11 +277 val_277 2008-04-08 11 +230 val_230 2008-04-08 11 +478 val_478 2008-04-08 11 +178 val_178 2008-04-08 11 +468 val_468 2008-04-08 11 +310 val_310 2008-04-08 11 +317 val_317 2008-04-08 11 +333 val_333 2008-04-08 11 +493 val_493 2008-04-08 11 +460 val_460 2008-04-08 11 +207 val_207 2008-04-08 11 +249 val_249 2008-04-08 11 +265 val_265 2008-04-08 11 +480 val_480 2008-04-08 11 +83 val_83 2008-04-08 11 +136 val_136 2008-04-08 11 +353 val_353 2008-04-08 11 +172 val_172 2008-04-08 11 +214 val_214 2008-04-08 11 +462 val_462 2008-04-08 11 +233 val_233 2008-04-08 11 +406 val_406 2008-04-08 11 +133 val_133 2008-04-08 11 +175 val_175 2008-04-08 11 +189 val_189 2008-04-08 11 +454 val_454 2008-04-08 11 +375 val_375 2008-04-08 11 +401 val_401 2008-04-08 11 +421 val_421 2008-04-08 11 +407 val_407 2008-04-08 11 +384 val_384 2008-04-08 11 +256 val_256 2008-04-08 11 +26 val_26 2008-04-08 11 +134 val_134 2008-04-08 11 +67 val_67 2008-04-08 11 +384 val_384 2008-04-08 11 +379 val_379 2008-04-08 11 +18 val_18 2008-04-08 11 +462 val_462 2008-04-08 11 +492 val_492 2008-04-08 11 +100 val_100 2008-04-08 11 +298 val_298 2008-04-08 11 +9 val_9 2008-04-08 11 +341 val_341 2008-04-08 11 +498 val_498 2008-04-08 11 +146 val_146 2008-04-08 11 +458 val_458 2008-04-08 11 +362 val_362 2008-04-08 11 +186 val_186 2008-04-08 11 +285 val_285 2008-04-08 11 +348 val_348 2008-04-08 11 +167 val_167 2008-04-08 11 +18 val_18 2008-04-08 11 +273 val_273 2008-04-08 11 +183 val_183 2008-04-08 11 +281 val_281 2008-04-08 11 +344 val_344 2008-04-08 11 +97 val_97 2008-04-08 11 +469 val_469 2008-04-08 11 +315 val_315 2008-04-08 11 +84 val_84 2008-04-08 11 +28 val_28 2008-04-08 11 +37 val_37 2008-04-08 11 +448 val_448 2008-04-08 11 +152 val_152 2008-04-08 11 +348 val_348 2008-04-08 11 +307 val_307 2008-04-08 11 +194 val_194 2008-04-08 11 +414 val_414 2008-04-08 11 +477 val_477 2008-04-08 11 +222 val_222 2008-04-08 11 +126 val_126 2008-04-08 11 +90 val_90 2008-04-08 11 +169 val_169 2008-04-08 11 +403 val_403 2008-04-08 11 +400 val_400 2008-04-08 11 +200 val_200 2008-04-08 11 +97 val_97 2008-04-08 11 +238 val_238 2008-04-08 12 +86 val_86 2008-04-08 12 +311 val_311 2008-04-08 12 +27 val_27 2008-04-08 12 +165 val_165 2008-04-08 12 +409 val_409 2008-04-08 12 +255 val_255 2008-04-08 12 +278 val_278 2008-04-08 12 +98 val_98 2008-04-08 12 +484 val_484 2008-04-08 12 +265 val_265 2008-04-08 12 +193 val_193 2008-04-08 12 +401 val_401 2008-04-08 12 +150 val_150 2008-04-08 12 +273 val_273 2008-04-08 12 +224 val_224 2008-04-08 12 +369 val_369 2008-04-08 12 +66 val_66 2008-04-08 12 +128 val_128 2008-04-08 12 +213 val_213 2008-04-08 12 +146 val_146 2008-04-08 12 +406 val_406 2008-04-08 12 +429 val_429 2008-04-08 12 +374 val_374 2008-04-08 12 +152 val_152 2008-04-08 12 +469 val_469 2008-04-08 12 +145 val_145 2008-04-08 12 +495 val_495 2008-04-08 12 +37 val_37 2008-04-08 12 +327 val_327 2008-04-08 12 +281 val_281 2008-04-08 12 +277 val_277 2008-04-08 12 +209 val_209 2008-04-08 12 +15 val_15 2008-04-08 12 +82 val_82 2008-04-08 12 +403 val_403 2008-04-08 12 +166 val_166 2008-04-08 12 +417 val_417 2008-04-08 12 +430 val_430 2008-04-08 12 +252 val_252 2008-04-08 12 +292 val_292 2008-04-08 12 +219 val_219 2008-04-08 12 +287 val_287 2008-04-08 12 +153 val_153 2008-04-08 12 +193 val_193 2008-04-08 12 +338 val_338 2008-04-08 12 +446 val_446 2008-04-08 12 +459 val_459 2008-04-08 12 +394 val_394 2008-04-08 12 +237 val_237 2008-04-08 12 +482 val_482 2008-04-08 12 +174 val_174 2008-04-08 12 +413 val_413 2008-04-08 12 +494 val_494 2008-04-08 12 +207 val_207 2008-04-08 12 +199 val_199 2008-04-08 12 +466 val_466 2008-04-08 12 +208 val_208 2008-04-08 12 +174 val_174 2008-04-08 12 +399 val_399 2008-04-08 12 +396 val_396 2008-04-08 12 +247 val_247 2008-04-08 12 +417 val_417 2008-04-08 12 +489 val_489 2008-04-08 12 +162 val_162 2008-04-08 12 +377 val_377 2008-04-08 12 +397 val_397 2008-04-08 12 +309 val_309 2008-04-08 12 +365 val_365 2008-04-08 12 +266 val_266 2008-04-08 12 +439 val_439 2008-04-08 12 +342 val_342 2008-04-08 12 +367 val_367 2008-04-08 12 +325 val_325 2008-04-08 12 +167 val_167 2008-04-08 12 +195 val_195 2008-04-08 12 +475 val_475 2008-04-08 12 +17 val_17 2008-04-08 12 +113 val_113 2008-04-08 12 +155 val_155 2008-04-08 12 +203 val_203 2008-04-08 12 +339 val_339 2008-04-08 12 +0 val_0 2008-04-08 12 +455 val_455 2008-04-08 12 +128 val_128 2008-04-08 12 +311 val_311 2008-04-08 12 +316 val_316 2008-04-08 12 +57 val_57 2008-04-08 12 +302 val_302 2008-04-08 12 +205 val_205 2008-04-08 12 +149 val_149 2008-04-08 12 +438 val_438 2008-04-08 12 +345 val_345 2008-04-08 12 +129 val_129 2008-04-08 12 +170 val_170 2008-04-08 12 +20 val_20 2008-04-08 12 +489 val_489 2008-04-08 12 +157 val_157 2008-04-08 12 +378 val_378 2008-04-08 12 +221 val_221 2008-04-08 12 +92 val_92 2008-04-08 12 +111 val_111 2008-04-08 12 +47 val_47 2008-04-08 12 +72 val_72 2008-04-08 12 +4 val_4 2008-04-08 12 +280 val_280 2008-04-08 12 +35 val_35 2008-04-08 12 +427 val_427 2008-04-08 12 +277 val_277 2008-04-08 12 +208 val_208 2008-04-08 12 +356 val_356 2008-04-08 12 +399 val_399 2008-04-08 12 +169 val_169 2008-04-08 12 +382 val_382 2008-04-08 12 +498 val_498 2008-04-08 12 +125 val_125 2008-04-08 12 +386 val_386 2008-04-08 12 +437 val_437 2008-04-08 12 +469 val_469 2008-04-08 12 +192 val_192 2008-04-08 12 +286 val_286 2008-04-08 12 +187 val_187 2008-04-08 12 +176 val_176 2008-04-08 12 +54 val_54 2008-04-08 12 +459 val_459 2008-04-08 12 +51 val_51 2008-04-08 12 +138 val_138 2008-04-08 12 +103 val_103 2008-04-08 12 +239 val_239 2008-04-08 12 +213 val_213 2008-04-08 12 +216 val_216 2008-04-08 12 +430 val_430 2008-04-08 12 +278 val_278 2008-04-08 12 +176 val_176 2008-04-08 12 +289 val_289 2008-04-08 12 +221 val_221 2008-04-08 12 +65 val_65 2008-04-08 12 +318 val_318 2008-04-08 12 +332 val_332 2008-04-08 12 +311 val_311 2008-04-08 12 +275 val_275 2008-04-08 12 +137 val_137 2008-04-08 12 +241 val_241 2008-04-08 12 +83 val_83 2008-04-08 12 +333 val_333 2008-04-08 12 +180 val_180 2008-04-08 12 +284 val_284 2008-04-08 12 +12 val_12 2008-04-08 12 +230 val_230 2008-04-08 12 +181 val_181 2008-04-08 12 +67 val_67 2008-04-08 12 +260 val_260 2008-04-08 12 +404 val_404 2008-04-08 12 +384 val_384 2008-04-08 12 +489 val_489 2008-04-08 12 +353 val_353 2008-04-08 12 +373 val_373 2008-04-08 12 +272 val_272 2008-04-08 12 +138 val_138 2008-04-08 12 +217 val_217 2008-04-08 12 +84 val_84 2008-04-08 12 +348 val_348 2008-04-08 12 +466 val_466 2008-04-08 12 +58 val_58 2008-04-08 12 +8 val_8 2008-04-08 12 +411 val_411 2008-04-08 12 +230 val_230 2008-04-08 12 +208 val_208 2008-04-08 12 +348 val_348 2008-04-08 12 +24 val_24 2008-04-08 12 +463 val_463 2008-04-08 12 +431 val_431 2008-04-08 12 +179 val_179 2008-04-08 12 +172 val_172 2008-04-08 12 +42 val_42 2008-04-08 12 +129 val_129 2008-04-08 12 +158 val_158 2008-04-08 12 +119 val_119 2008-04-08 12 +496 val_496 2008-04-08 12 +0 val_0 2008-04-08 12 +322 val_322 2008-04-08 12 +197 val_197 2008-04-08 12 +468 val_468 2008-04-08 12 +393 val_393 2008-04-08 12 +454 val_454 2008-04-08 12 +100 val_100 2008-04-08 12 +298 val_298 2008-04-08 12 +199 val_199 2008-04-08 12 +191 val_191 2008-04-08 12 +418 val_418 2008-04-08 12 +96 val_96 2008-04-08 12 +26 val_26 2008-04-08 12 +165 val_165 2008-04-08 12 +327 val_327 2008-04-08 12 +230 val_230 2008-04-08 12 +205 val_205 2008-04-08 12 +120 val_120 2008-04-08 12 +131 val_131 2008-04-08 12 +51 val_51 2008-04-08 12 +404 val_404 2008-04-08 12 +43 val_43 2008-04-08 12 +436 val_436 2008-04-08 12 +156 val_156 2008-04-08 12 +469 val_469 2008-04-08 12 +468 val_468 2008-04-08 12 +308 val_308 2008-04-08 12 +95 val_95 2008-04-08 12 +196 val_196 2008-04-08 12 +288 val_288 2008-04-08 12 +481 val_481 2008-04-08 12 +457 val_457 2008-04-08 12 +98 val_98 2008-04-08 12 +282 val_282 2008-04-08 12 +197 val_197 2008-04-08 12 +187 val_187 2008-04-08 12 +318 val_318 2008-04-08 12 +318 val_318 2008-04-08 12 +409 val_409 2008-04-08 12 +470 val_470 2008-04-08 12 +137 val_137 2008-04-08 12 +369 val_369 2008-04-08 12 +316 val_316 2008-04-08 12 +169 val_169 2008-04-08 12 +413 val_413 2008-04-08 12 +85 val_85 2008-04-08 12 +77 val_77 2008-04-08 12 +0 val_0 2008-04-08 12 +490 val_490 2008-04-08 12 +87 val_87 2008-04-08 12 +364 val_364 2008-04-08 12 +179 val_179 2008-04-08 12 +118 val_118 2008-04-08 12 +134 val_134 2008-04-08 12 +395 val_395 2008-04-08 12 +282 val_282 2008-04-08 12 +138 val_138 2008-04-08 12 +238 val_238 2008-04-08 12 +419 val_419 2008-04-08 12 +15 val_15 2008-04-08 12 +118 val_118 2008-04-08 12 +72 val_72 2008-04-08 12 +90 val_90 2008-04-08 12 +307 val_307 2008-04-08 12 +19 val_19 2008-04-08 12 +435 val_435 2008-04-08 12 +10 val_10 2008-04-08 12 +277 val_277 2008-04-08 12 +273 val_273 2008-04-08 12 +306 val_306 2008-04-08 12 +224 val_224 2008-04-08 12 +309 val_309 2008-04-08 12 +389 val_389 2008-04-08 12 +327 val_327 2008-04-08 12 +242 val_242 2008-04-08 12 +369 val_369 2008-04-08 12 +392 val_392 2008-04-08 12 +272 val_272 2008-04-08 12 +331 val_331 2008-04-08 12 +401 val_401 2008-04-08 12 +242 val_242 2008-04-08 12 +452 val_452 2008-04-08 12 +177 val_177 2008-04-08 12 +226 val_226 2008-04-08 12 +5 val_5 2008-04-08 12 +497 val_497 2008-04-08 12 +402 val_402 2008-04-08 12 +396 val_396 2008-04-08 12 +317 val_317 2008-04-08 12 +395 val_395 2008-04-08 12 +58 val_58 2008-04-08 12 +35 val_35 2008-04-08 12 +336 val_336 2008-04-08 12 +95 val_95 2008-04-08 12 +11 val_11 2008-04-08 12 +168 val_168 2008-04-08 12 +34 val_34 2008-04-08 12 +229 val_229 2008-04-08 12 +233 val_233 2008-04-08 12 +143 val_143 2008-04-08 12 +472 val_472 2008-04-08 12 +322 val_322 2008-04-08 12 +498 val_498 2008-04-08 12 +160 val_160 2008-04-08 12 +195 val_195 2008-04-08 12 +42 val_42 2008-04-08 12 +321 val_321 2008-04-08 12 +430 val_430 2008-04-08 12 +119 val_119 2008-04-08 12 +489 val_489 2008-04-08 12 +458 val_458 2008-04-08 12 +78 val_78 2008-04-08 12 +76 val_76 2008-04-08 12 +41 val_41 2008-04-08 12 +223 val_223 2008-04-08 12 +492 val_492 2008-04-08 12 +149 val_149 2008-04-08 12 +449 val_449 2008-04-08 12 +218 val_218 2008-04-08 12 +228 val_228 2008-04-08 12 +138 val_138 2008-04-08 12 +453 val_453 2008-04-08 12 +30 val_30 2008-04-08 12 +209 val_209 2008-04-08 12 +64 val_64 2008-04-08 12 +468 val_468 2008-04-08 12 +76 val_76 2008-04-08 12 +74 val_74 2008-04-08 12 +342 val_342 2008-04-08 12 +69 val_69 2008-04-08 12 +230 val_230 2008-04-08 12 +33 val_33 2008-04-08 12 +368 val_368 2008-04-08 12 +103 val_103 2008-04-08 12 +296 val_296 2008-04-08 12 +113 val_113 2008-04-08 12 +216 val_216 2008-04-08 12 +367 val_367 2008-04-08 12 +344 val_344 2008-04-08 12 +167 val_167 2008-04-08 12 +274 val_274 2008-04-08 12 +219 val_219 2008-04-08 12 +239 val_239 2008-04-08 12 +485 val_485 2008-04-08 12 +116 val_116 2008-04-08 12 +223 val_223 2008-04-08 12 +256 val_256 2008-04-08 12 +263 val_263 2008-04-08 12 +70 val_70 2008-04-08 12 +487 val_487 2008-04-08 12 +480 val_480 2008-04-08 12 +401 val_401 2008-04-08 12 +288 val_288 2008-04-08 12 +191 val_191 2008-04-08 12 +5 val_5 2008-04-08 12 +244 val_244 2008-04-08 12 +438 val_438 2008-04-08 12 +128 val_128 2008-04-08 12 +467 val_467 2008-04-08 12 +432 val_432 2008-04-08 12 +202 val_202 2008-04-08 12 +316 val_316 2008-04-08 12 +229 val_229 2008-04-08 12 +469 val_469 2008-04-08 12 +463 val_463 2008-04-08 12 +280 val_280 2008-04-08 12 +2 val_2 2008-04-08 12 +35 val_35 2008-04-08 12 +283 val_283 2008-04-08 12 +331 val_331 2008-04-08 12 +235 val_235 2008-04-08 12 +80 val_80 2008-04-08 12 +44 val_44 2008-04-08 12 +193 val_193 2008-04-08 12 +321 val_321 2008-04-08 12 +335 val_335 2008-04-08 12 +104 val_104 2008-04-08 12 +466 val_466 2008-04-08 12 +366 val_366 2008-04-08 12 +175 val_175 2008-04-08 12 +403 val_403 2008-04-08 12 +483 val_483 2008-04-08 12 +53 val_53 2008-04-08 12 +105 val_105 2008-04-08 12 +257 val_257 2008-04-08 12 +406 val_406 2008-04-08 12 +409 val_409 2008-04-08 12 +190 val_190 2008-04-08 12 +406 val_406 2008-04-08 12 +401 val_401 2008-04-08 12 +114 val_114 2008-04-08 12 +258 val_258 2008-04-08 12 +90 val_90 2008-04-08 12 +203 val_203 2008-04-08 12 +262 val_262 2008-04-08 12 +348 val_348 2008-04-08 12 +424 val_424 2008-04-08 12 +12 val_12 2008-04-08 12 +396 val_396 2008-04-08 12 +201 val_201 2008-04-08 12 +217 val_217 2008-04-08 12 +164 val_164 2008-04-08 12 +431 val_431 2008-04-08 12 +454 val_454 2008-04-08 12 +478 val_478 2008-04-08 12 +298 val_298 2008-04-08 12 +125 val_125 2008-04-08 12 +431 val_431 2008-04-08 12 +164 val_164 2008-04-08 12 +424 val_424 2008-04-08 12 +187 val_187 2008-04-08 12 +382 val_382 2008-04-08 12 +5 val_5 2008-04-08 12 +70 val_70 2008-04-08 12 +397 val_397 2008-04-08 12 +480 val_480 2008-04-08 12 +291 val_291 2008-04-08 12 +24 val_24 2008-04-08 12 +351 val_351 2008-04-08 12 +255 val_255 2008-04-08 12 +104 val_104 2008-04-08 12 +70 val_70 2008-04-08 12 +163 val_163 2008-04-08 12 +438 val_438 2008-04-08 12 +119 val_119 2008-04-08 12 +414 val_414 2008-04-08 12 +200 val_200 2008-04-08 12 +491 val_491 2008-04-08 12 +237 val_237 2008-04-08 12 +439 val_439 2008-04-08 12 +360 val_360 2008-04-08 12 +248 val_248 2008-04-08 12 +479 val_479 2008-04-08 12 +305 val_305 2008-04-08 12 +417 val_417 2008-04-08 12 +199 val_199 2008-04-08 12 +444 val_444 2008-04-08 12 +120 val_120 2008-04-08 12 +429 val_429 2008-04-08 12 +169 val_169 2008-04-08 12 +443 val_443 2008-04-08 12 +323 val_323 2008-04-08 12 +325 val_325 2008-04-08 12 +277 val_277 2008-04-08 12 +230 val_230 2008-04-08 12 +478 val_478 2008-04-08 12 +178 val_178 2008-04-08 12 +468 val_468 2008-04-08 12 +310 val_310 2008-04-08 12 +317 val_317 2008-04-08 12 +333 val_333 2008-04-08 12 +493 val_493 2008-04-08 12 +460 val_460 2008-04-08 12 +207 val_207 2008-04-08 12 +249 val_249 2008-04-08 12 +265 val_265 2008-04-08 12 +480 val_480 2008-04-08 12 +83 val_83 2008-04-08 12 +136 val_136 2008-04-08 12 +353 val_353 2008-04-08 12 +172 val_172 2008-04-08 12 +214 val_214 2008-04-08 12 +462 val_462 2008-04-08 12 +233 val_233 2008-04-08 12 +406 val_406 2008-04-08 12 +133 val_133 2008-04-08 12 +175 val_175 2008-04-08 12 +189 val_189 2008-04-08 12 +454 val_454 2008-04-08 12 +375 val_375 2008-04-08 12 +401 val_401 2008-04-08 12 +421 val_421 2008-04-08 12 +407 val_407 2008-04-08 12 +384 val_384 2008-04-08 12 +256 val_256 2008-04-08 12 +26 val_26 2008-04-08 12 +134 val_134 2008-04-08 12 +67 val_67 2008-04-08 12 +384 val_384 2008-04-08 12 +379 val_379 2008-04-08 12 +18 val_18 2008-04-08 12 +462 val_462 2008-04-08 12 +492 val_492 2008-04-08 12 +100 val_100 2008-04-08 12 +298 val_298 2008-04-08 12 +9 val_9 2008-04-08 12 +341 val_341 2008-04-08 12 +498 val_498 2008-04-08 12 +146 val_146 2008-04-08 12 +458 val_458 2008-04-08 12 +362 val_362 2008-04-08 12 +186 val_186 2008-04-08 12 +285 val_285 2008-04-08 12 +348 val_348 2008-04-08 12 +167 val_167 2008-04-08 12 +18 val_18 2008-04-08 12 +273 val_273 2008-04-08 12 +183 val_183 2008-04-08 12 +281 val_281 2008-04-08 12 +344 val_344 2008-04-08 12 +97 val_97 2008-04-08 12 +469 val_469 2008-04-08 12 +315 val_315 2008-04-08 12 +84 val_84 2008-04-08 12 +28 val_28 2008-04-08 12 +37 val_37 2008-04-08 12 +448 val_448 2008-04-08 12 +152 val_152 2008-04-08 12 +348 val_348 2008-04-08 12 +307 val_307 2008-04-08 12 +194 val_194 2008-04-08 12 +414 val_414 2008-04-08 12 +477 val_477 2008-04-08 12 +222 val_222 2008-04-08 12 +126 val_126 2008-04-08 12 +90 val_90 2008-04-08 12 +169 val_169 2008-04-08 12 +403 val_403 2008-04-08 12 +400 val_400 2008-04-08 12 +200 val_200 2008-04-08 12 +97 val_97 2008-04-08 12 +PREHOOK: query: select * from nzhang_part2 where ds is not null and hr is not null +PREHOOK: type: QUERY +PREHOOK: Input: default@nzhang_part2@ds=2008-12-31/hr=11 +PREHOOK: Input: default@nzhang_part2@ds=2008-12-31/hr=12 +PREHOOK: Output: file:/tmp/aaly/hive_2010-08-09_13-10-50_360_1986744011429301882/-mr-10000 +POSTHOOK: query: select * from nzhang_part2 where ds is not null and hr is not null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@nzhang_part2@ds=2008-12-31/hr=11 +POSTHOOK: Input: default@nzhang_part2@ds=2008-12-31/hr=12 +POSTHOOK: Output: file:/tmp/aaly/hive_2010-08-09_13-10-50_360_1986744011429301882/-mr-10000 +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +238 val_238 2008-12-31 11 +86 val_86 2008-12-31 11 +311 val_311 2008-12-31 11 +27 val_27 2008-12-31 11 +165 val_165 2008-12-31 11 +409 val_409 2008-12-31 11 +255 val_255 2008-12-31 11 +278 val_278 2008-12-31 11 +98 val_98 2008-12-31 11 +484 val_484 2008-12-31 11 +265 val_265 2008-12-31 11 +193 val_193 2008-12-31 11 +401 val_401 2008-12-31 11 +150 val_150 2008-12-31 11 +273 val_273 2008-12-31 11 +224 val_224 2008-12-31 11 +369 val_369 2008-12-31 11 +66 val_66 2008-12-31 11 +128 val_128 2008-12-31 11 +213 val_213 2008-12-31 11 +146 val_146 2008-12-31 11 +406 val_406 2008-12-31 11 +429 val_429 2008-12-31 11 +374 val_374 2008-12-31 11 +152 val_152 2008-12-31 11 +469 val_469 2008-12-31 11 +145 val_145 2008-12-31 11 +495 val_495 2008-12-31 11 +37 val_37 2008-12-31 11 +327 val_327 2008-12-31 11 +281 val_281 2008-12-31 11 +277 val_277 2008-12-31 11 +209 val_209 2008-12-31 11 +15 val_15 2008-12-31 11 +82 val_82 2008-12-31 11 +403 val_403 2008-12-31 11 +166 val_166 2008-12-31 11 +417 val_417 2008-12-31 11 +430 val_430 2008-12-31 11 +252 val_252 2008-12-31 11 +292 val_292 2008-12-31 11 +219 val_219 2008-12-31 11 +287 val_287 2008-12-31 11 +153 val_153 2008-12-31 11 +193 val_193 2008-12-31 11 +338 val_338 2008-12-31 11 +446 val_446 2008-12-31 11 +459 val_459 2008-12-31 11 +394 val_394 2008-12-31 11 +237 val_237 2008-12-31 11 +482 val_482 2008-12-31 11 +174 val_174 2008-12-31 11 +413 val_413 2008-12-31 11 +494 val_494 2008-12-31 11 +207 val_207 2008-12-31 11 +199 val_199 2008-12-31 11 +466 val_466 2008-12-31 11 +208 val_208 2008-12-31 11 +174 val_174 2008-12-31 11 +399 val_399 2008-12-31 11 +396 val_396 2008-12-31 11 +247 val_247 2008-12-31 11 +417 val_417 2008-12-31 11 +489 val_489 2008-12-31 11 +162 val_162 2008-12-31 11 +377 val_377 2008-12-31 11 +397 val_397 2008-12-31 11 +309 val_309 2008-12-31 11 +365 val_365 2008-12-31 11 +266 val_266 2008-12-31 11 +439 val_439 2008-12-31 11 +342 val_342 2008-12-31 11 +367 val_367 2008-12-31 11 +325 val_325 2008-12-31 11 +167 val_167 2008-12-31 11 +195 val_195 2008-12-31 11 +475 val_475 2008-12-31 11 +17 val_17 2008-12-31 11 +113 val_113 2008-12-31 11 +155 val_155 2008-12-31 11 +203 val_203 2008-12-31 11 +339 val_339 2008-12-31 11 +0 val_0 2008-12-31 11 +455 val_455 2008-12-31 11 +128 val_128 2008-12-31 11 +311 val_311 2008-12-31 11 +316 val_316 2008-12-31 11 +57 val_57 2008-12-31 11 +302 val_302 2008-12-31 11 +205 val_205 2008-12-31 11 +149 val_149 2008-12-31 11 +438 val_438 2008-12-31 11 +345 val_345 2008-12-31 11 +129 val_129 2008-12-31 11 +170 val_170 2008-12-31 11 +20 val_20 2008-12-31 11 +489 val_489 2008-12-31 11 +157 val_157 2008-12-31 11 +378 val_378 2008-12-31 11 +221 val_221 2008-12-31 11 +92 val_92 2008-12-31 11 +111 val_111 2008-12-31 11 +47 val_47 2008-12-31 11 +72 val_72 2008-12-31 11 +4 val_4 2008-12-31 11 +280 val_280 2008-12-31 11 +35 val_35 2008-12-31 11 +427 val_427 2008-12-31 11 +277 val_277 2008-12-31 11 +208 val_208 2008-12-31 11 +356 val_356 2008-12-31 11 +399 val_399 2008-12-31 11 +169 val_169 2008-12-31 11 +382 val_382 2008-12-31 11 +498 val_498 2008-12-31 11 +125 val_125 2008-12-31 11 +386 val_386 2008-12-31 11 +437 val_437 2008-12-31 11 +469 val_469 2008-12-31 11 +192 val_192 2008-12-31 11 +286 val_286 2008-12-31 11 +187 val_187 2008-12-31 11 +176 val_176 2008-12-31 11 +54 val_54 2008-12-31 11 +459 val_459 2008-12-31 11 +51 val_51 2008-12-31 11 +138 val_138 2008-12-31 11 +103 val_103 2008-12-31 11 +239 val_239 2008-12-31 11 +213 val_213 2008-12-31 11 +216 val_216 2008-12-31 11 +430 val_430 2008-12-31 11 +278 val_278 2008-12-31 11 +176 val_176 2008-12-31 11 +289 val_289 2008-12-31 11 +221 val_221 2008-12-31 11 +65 val_65 2008-12-31 11 +318 val_318 2008-12-31 11 +332 val_332 2008-12-31 11 +311 val_311 2008-12-31 11 +275 val_275 2008-12-31 11 +137 val_137 2008-12-31 11 +241 val_241 2008-12-31 11 +83 val_83 2008-12-31 11 +333 val_333 2008-12-31 11 +180 val_180 2008-12-31 11 +284 val_284 2008-12-31 11 +12 val_12 2008-12-31 11 +230 val_230 2008-12-31 11 +181 val_181 2008-12-31 11 +67 val_67 2008-12-31 11 +260 val_260 2008-12-31 11 +404 val_404 2008-12-31 11 +384 val_384 2008-12-31 11 +489 val_489 2008-12-31 11 +353 val_353 2008-12-31 11 +373 val_373 2008-12-31 11 +272 val_272 2008-12-31 11 +138 val_138 2008-12-31 11 +217 val_217 2008-12-31 11 +84 val_84 2008-12-31 11 +348 val_348 2008-12-31 11 +466 val_466 2008-12-31 11 +58 val_58 2008-12-31 11 +8 val_8 2008-12-31 11 +411 val_411 2008-12-31 11 +230 val_230 2008-12-31 11 +208 val_208 2008-12-31 11 +348 val_348 2008-12-31 11 +24 val_24 2008-12-31 11 +463 val_463 2008-12-31 11 +431 val_431 2008-12-31 11 +179 val_179 2008-12-31 11 +172 val_172 2008-12-31 11 +42 val_42 2008-12-31 11 +129 val_129 2008-12-31 11 +158 val_158 2008-12-31 11 +119 val_119 2008-12-31 11 +496 val_496 2008-12-31 11 +0 val_0 2008-12-31 11 +322 val_322 2008-12-31 11 +197 val_197 2008-12-31 11 +468 val_468 2008-12-31 11 +393 val_393 2008-12-31 11 +454 val_454 2008-12-31 11 +100 val_100 2008-12-31 11 +298 val_298 2008-12-31 11 +199 val_199 2008-12-31 11 +191 val_191 2008-12-31 11 +418 val_418 2008-12-31 11 +96 val_96 2008-12-31 11 +26 val_26 2008-12-31 11 +165 val_165 2008-12-31 11 +327 val_327 2008-12-31 11 +230 val_230 2008-12-31 11 +205 val_205 2008-12-31 11 +120 val_120 2008-12-31 11 +131 val_131 2008-12-31 11 +51 val_51 2008-12-31 11 +404 val_404 2008-12-31 11 +43 val_43 2008-12-31 11 +436 val_436 2008-12-31 11 +156 val_156 2008-12-31 11 +469 val_469 2008-12-31 11 +468 val_468 2008-12-31 11 +308 val_308 2008-12-31 11 +95 val_95 2008-12-31 11 +196 val_196 2008-12-31 11 +288 val_288 2008-12-31 11 +481 val_481 2008-12-31 11 +457 val_457 2008-12-31 11 +98 val_98 2008-12-31 11 +282 val_282 2008-12-31 11 +197 val_197 2008-12-31 11 +187 val_187 2008-12-31 11 +318 val_318 2008-12-31 11 +318 val_318 2008-12-31 11 +409 val_409 2008-12-31 11 +470 val_470 2008-12-31 11 +137 val_137 2008-12-31 11 +369 val_369 2008-12-31 11 +316 val_316 2008-12-31 11 +169 val_169 2008-12-31 11 +413 val_413 2008-12-31 11 +85 val_85 2008-12-31 11 +77 val_77 2008-12-31 11 +0 val_0 2008-12-31 11 +490 val_490 2008-12-31 11 +87 val_87 2008-12-31 11 +364 val_364 2008-12-31 11 +179 val_179 2008-12-31 11 +118 val_118 2008-12-31 11 +134 val_134 2008-12-31 11 +395 val_395 2008-12-31 11 +282 val_282 2008-12-31 11 +138 val_138 2008-12-31 11 +238 val_238 2008-12-31 11 +419 val_419 2008-12-31 11 +15 val_15 2008-12-31 11 +118 val_118 2008-12-31 11 +72 val_72 2008-12-31 11 +90 val_90 2008-12-31 11 +307 val_307 2008-12-31 11 +19 val_19 2008-12-31 11 +435 val_435 2008-12-31 11 +10 val_10 2008-12-31 11 +277 val_277 2008-12-31 11 +273 val_273 2008-12-31 11 +306 val_306 2008-12-31 11 +224 val_224 2008-12-31 11 +309 val_309 2008-12-31 11 +389 val_389 2008-12-31 11 +327 val_327 2008-12-31 11 +242 val_242 2008-12-31 11 +369 val_369 2008-12-31 11 +392 val_392 2008-12-31 11 +272 val_272 2008-12-31 11 +331 val_331 2008-12-31 11 +401 val_401 2008-12-31 11 +242 val_242 2008-12-31 11 +452 val_452 2008-12-31 11 +177 val_177 2008-12-31 11 +226 val_226 2008-12-31 11 +5 val_5 2008-12-31 11 +497 val_497 2008-12-31 11 +402 val_402 2008-12-31 11 +396 val_396 2008-12-31 11 +317 val_317 2008-12-31 11 +395 val_395 2008-12-31 11 +58 val_58 2008-12-31 11 +35 val_35 2008-12-31 11 +336 val_336 2008-12-31 11 +95 val_95 2008-12-31 11 +11 val_11 2008-12-31 11 +168 val_168 2008-12-31 11 +34 val_34 2008-12-31 11 +229 val_229 2008-12-31 11 +233 val_233 2008-12-31 11 +143 val_143 2008-12-31 11 +472 val_472 2008-12-31 11 +322 val_322 2008-12-31 11 +498 val_498 2008-12-31 11 +160 val_160 2008-12-31 11 +195 val_195 2008-12-31 11 +42 val_42 2008-12-31 11 +321 val_321 2008-12-31 11 +430 val_430 2008-12-31 11 +119 val_119 2008-12-31 11 +489 val_489 2008-12-31 11 +458 val_458 2008-12-31 11 +78 val_78 2008-12-31 11 +76 val_76 2008-12-31 11 +41 val_41 2008-12-31 11 +223 val_223 2008-12-31 11 +492 val_492 2008-12-31 11 +149 val_149 2008-12-31 11 +449 val_449 2008-12-31 11 +218 val_218 2008-12-31 11 +228 val_228 2008-12-31 11 +138 val_138 2008-12-31 11 +453 val_453 2008-12-31 11 +30 val_30 2008-12-31 11 +209 val_209 2008-12-31 11 +64 val_64 2008-12-31 11 +468 val_468 2008-12-31 11 +76 val_76 2008-12-31 11 +74 val_74 2008-12-31 11 +342 val_342 2008-12-31 11 +69 val_69 2008-12-31 11 +230 val_230 2008-12-31 11 +33 val_33 2008-12-31 11 +368 val_368 2008-12-31 11 +103 val_103 2008-12-31 11 +296 val_296 2008-12-31 11 +113 val_113 2008-12-31 11 +216 val_216 2008-12-31 11 +367 val_367 2008-12-31 11 +344 val_344 2008-12-31 11 +167 val_167 2008-12-31 11 +274 val_274 2008-12-31 11 +219 val_219 2008-12-31 11 +239 val_239 2008-12-31 11 +485 val_485 2008-12-31 11 +116 val_116 2008-12-31 11 +223 val_223 2008-12-31 11 +256 val_256 2008-12-31 11 +263 val_263 2008-12-31 11 +70 val_70 2008-12-31 11 +487 val_487 2008-12-31 11 +480 val_480 2008-12-31 11 +401 val_401 2008-12-31 11 +288 val_288 2008-12-31 11 +191 val_191 2008-12-31 11 +5 val_5 2008-12-31 11 +244 val_244 2008-12-31 11 +438 val_438 2008-12-31 11 +128 val_128 2008-12-31 11 +467 val_467 2008-12-31 11 +432 val_432 2008-12-31 11 +202 val_202 2008-12-31 11 +316 val_316 2008-12-31 11 +229 val_229 2008-12-31 11 +469 val_469 2008-12-31 11 +463 val_463 2008-12-31 11 +280 val_280 2008-12-31 11 +2 val_2 2008-12-31 11 +35 val_35 2008-12-31 11 +283 val_283 2008-12-31 11 +331 val_331 2008-12-31 11 +235 val_235 2008-12-31 11 +80 val_80 2008-12-31 11 +44 val_44 2008-12-31 11 +193 val_193 2008-12-31 11 +321 val_321 2008-12-31 11 +335 val_335 2008-12-31 11 +104 val_104 2008-12-31 11 +466 val_466 2008-12-31 11 +366 val_366 2008-12-31 11 +175 val_175 2008-12-31 11 +403 val_403 2008-12-31 11 +483 val_483 2008-12-31 11 +53 val_53 2008-12-31 11 +105 val_105 2008-12-31 11 +257 val_257 2008-12-31 11 +406 val_406 2008-12-31 11 +409 val_409 2008-12-31 11 +190 val_190 2008-12-31 11 +406 val_406 2008-12-31 11 +401 val_401 2008-12-31 11 +114 val_114 2008-12-31 11 +258 val_258 2008-12-31 11 +90 val_90 2008-12-31 11 +203 val_203 2008-12-31 11 +262 val_262 2008-12-31 11 +348 val_348 2008-12-31 11 +424 val_424 2008-12-31 11 +12 val_12 2008-12-31 11 +396 val_396 2008-12-31 11 +201 val_201 2008-12-31 11 +217 val_217 2008-12-31 11 +164 val_164 2008-12-31 11 +431 val_431 2008-12-31 11 +454 val_454 2008-12-31 11 +478 val_478 2008-12-31 11 +298 val_298 2008-12-31 11 +125 val_125 2008-12-31 11 +431 val_431 2008-12-31 11 +164 val_164 2008-12-31 11 +424 val_424 2008-12-31 11 +187 val_187 2008-12-31 11 +382 val_382 2008-12-31 11 +5 val_5 2008-12-31 11 +70 val_70 2008-12-31 11 +397 val_397 2008-12-31 11 +480 val_480 2008-12-31 11 +291 val_291 2008-12-31 11 +24 val_24 2008-12-31 11 +351 val_351 2008-12-31 11 +255 val_255 2008-12-31 11 +104 val_104 2008-12-31 11 +70 val_70 2008-12-31 11 +163 val_163 2008-12-31 11 +438 val_438 2008-12-31 11 +119 val_119 2008-12-31 11 +414 val_414 2008-12-31 11 +200 val_200 2008-12-31 11 +491 val_491 2008-12-31 11 +237 val_237 2008-12-31 11 +439 val_439 2008-12-31 11 +360 val_360 2008-12-31 11 +248 val_248 2008-12-31 11 +479 val_479 2008-12-31 11 +305 val_305 2008-12-31 11 +417 val_417 2008-12-31 11 +199 val_199 2008-12-31 11 +444 val_444 2008-12-31 11 +120 val_120 2008-12-31 11 +429 val_429 2008-12-31 11 +169 val_169 2008-12-31 11 +443 val_443 2008-12-31 11 +323 val_323 2008-12-31 11 +325 val_325 2008-12-31 11 +277 val_277 2008-12-31 11 +230 val_230 2008-12-31 11 +478 val_478 2008-12-31 11 +178 val_178 2008-12-31 11 +468 val_468 2008-12-31 11 +310 val_310 2008-12-31 11 +317 val_317 2008-12-31 11 +333 val_333 2008-12-31 11 +493 val_493 2008-12-31 11 +460 val_460 2008-12-31 11 +207 val_207 2008-12-31 11 +249 val_249 2008-12-31 11 +265 val_265 2008-12-31 11 +480 val_480 2008-12-31 11 +83 val_83 2008-12-31 11 +136 val_136 2008-12-31 11 +353 val_353 2008-12-31 11 +172 val_172 2008-12-31 11 +214 val_214 2008-12-31 11 +462 val_462 2008-12-31 11 +233 val_233 2008-12-31 11 +406 val_406 2008-12-31 11 +133 val_133 2008-12-31 11 +175 val_175 2008-12-31 11 +189 val_189 2008-12-31 11 +454 val_454 2008-12-31 11 +375 val_375 2008-12-31 11 +401 val_401 2008-12-31 11 +421 val_421 2008-12-31 11 +407 val_407 2008-12-31 11 +384 val_384 2008-12-31 11 +256 val_256 2008-12-31 11 +26 val_26 2008-12-31 11 +134 val_134 2008-12-31 11 +67 val_67 2008-12-31 11 +384 val_384 2008-12-31 11 +379 val_379 2008-12-31 11 +18 val_18 2008-12-31 11 +462 val_462 2008-12-31 11 +492 val_492 2008-12-31 11 +100 val_100 2008-12-31 11 +298 val_298 2008-12-31 11 +9 val_9 2008-12-31 11 +341 val_341 2008-12-31 11 +498 val_498 2008-12-31 11 +146 val_146 2008-12-31 11 +458 val_458 2008-12-31 11 +362 val_362 2008-12-31 11 +186 val_186 2008-12-31 11 +285 val_285 2008-12-31 11 +348 val_348 2008-12-31 11 +167 val_167 2008-12-31 11 +18 val_18 2008-12-31 11 +273 val_273 2008-12-31 11 +183 val_183 2008-12-31 11 +281 val_281 2008-12-31 11 +344 val_344 2008-12-31 11 +97 val_97 2008-12-31 11 +469 val_469 2008-12-31 11 +315 val_315 2008-12-31 11 +84 val_84 2008-12-31 11 +28 val_28 2008-12-31 11 +37 val_37 2008-12-31 11 +448 val_448 2008-12-31 11 +152 val_152 2008-12-31 11 +348 val_348 2008-12-31 11 +307 val_307 2008-12-31 11 +194 val_194 2008-12-31 11 +414 val_414 2008-12-31 11 +477 val_477 2008-12-31 11 +222 val_222 2008-12-31 11 +126 val_126 2008-12-31 11 +90 val_90 2008-12-31 11 +169 val_169 2008-12-31 11 +403 val_403 2008-12-31 11 +400 val_400 2008-12-31 11 +200 val_200 2008-12-31 11 +97 val_97 2008-12-31 11 +238 val_238 2008-12-31 12 +86 val_86 2008-12-31 12 +311 val_311 2008-12-31 12 +27 val_27 2008-12-31 12 +165 val_165 2008-12-31 12 +409 val_409 2008-12-31 12 +255 val_255 2008-12-31 12 +278 val_278 2008-12-31 12 +98 val_98 2008-12-31 12 +484 val_484 2008-12-31 12 +265 val_265 2008-12-31 12 +193 val_193 2008-12-31 12 +401 val_401 2008-12-31 12 +150 val_150 2008-12-31 12 +273 val_273 2008-12-31 12 +224 val_224 2008-12-31 12 +369 val_369 2008-12-31 12 +66 val_66 2008-12-31 12 +128 val_128 2008-12-31 12 +213 val_213 2008-12-31 12 +146 val_146 2008-12-31 12 +406 val_406 2008-12-31 12 +429 val_429 2008-12-31 12 +374 val_374 2008-12-31 12 +152 val_152 2008-12-31 12 +469 val_469 2008-12-31 12 +145 val_145 2008-12-31 12 +495 val_495 2008-12-31 12 +37 val_37 2008-12-31 12 +327 val_327 2008-12-31 12 +281 val_281 2008-12-31 12 +277 val_277 2008-12-31 12 +209 val_209 2008-12-31 12 +15 val_15 2008-12-31 12 +82 val_82 2008-12-31 12 +403 val_403 2008-12-31 12 +166 val_166 2008-12-31 12 +417 val_417 2008-12-31 12 +430 val_430 2008-12-31 12 +252 val_252 2008-12-31 12 +292 val_292 2008-12-31 12 +219 val_219 2008-12-31 12 +287 val_287 2008-12-31 12 +153 val_153 2008-12-31 12 +193 val_193 2008-12-31 12 +338 val_338 2008-12-31 12 +446 val_446 2008-12-31 12 +459 val_459 2008-12-31 12 +394 val_394 2008-12-31 12 +237 val_237 2008-12-31 12 +482 val_482 2008-12-31 12 +174 val_174 2008-12-31 12 +413 val_413 2008-12-31 12 +494 val_494 2008-12-31 12 +207 val_207 2008-12-31 12 +199 val_199 2008-12-31 12 +466 val_466 2008-12-31 12 +208 val_208 2008-12-31 12 +174 val_174 2008-12-31 12 +399 val_399 2008-12-31 12 +396 val_396 2008-12-31 12 +247 val_247 2008-12-31 12 +417 val_417 2008-12-31 12 +489 val_489 2008-12-31 12 +162 val_162 2008-12-31 12 +377 val_377 2008-12-31 12 +397 val_397 2008-12-31 12 +309 val_309 2008-12-31 12 +365 val_365 2008-12-31 12 +266 val_266 2008-12-31 12 +439 val_439 2008-12-31 12 +342 val_342 2008-12-31 12 +367 val_367 2008-12-31 12 +325 val_325 2008-12-31 12 +167 val_167 2008-12-31 12 +195 val_195 2008-12-31 12 +475 val_475 2008-12-31 12 +17 val_17 2008-12-31 12 +113 val_113 2008-12-31 12 +155 val_155 2008-12-31 12 +203 val_203 2008-12-31 12 +339 val_339 2008-12-31 12 +0 val_0 2008-12-31 12 +455 val_455 2008-12-31 12 +128 val_128 2008-12-31 12 +311 val_311 2008-12-31 12 +316 val_316 2008-12-31 12 +57 val_57 2008-12-31 12 +302 val_302 2008-12-31 12 +205 val_205 2008-12-31 12 +149 val_149 2008-12-31 12 +438 val_438 2008-12-31 12 +345 val_345 2008-12-31 12 +129 val_129 2008-12-31 12 +170 val_170 2008-12-31 12 +20 val_20 2008-12-31 12 +489 val_489 2008-12-31 12 +157 val_157 2008-12-31 12 +378 val_378 2008-12-31 12 +221 val_221 2008-12-31 12 +92 val_92 2008-12-31 12 +111 val_111 2008-12-31 12 +47 val_47 2008-12-31 12 +72 val_72 2008-12-31 12 +4 val_4 2008-12-31 12 +280 val_280 2008-12-31 12 +35 val_35 2008-12-31 12 +427 val_427 2008-12-31 12 +277 val_277 2008-12-31 12 +208 val_208 2008-12-31 12 +356 val_356 2008-12-31 12 +399 val_399 2008-12-31 12 +169 val_169 2008-12-31 12 +382 val_382 2008-12-31 12 +498 val_498 2008-12-31 12 +125 val_125 2008-12-31 12 +386 val_386 2008-12-31 12 +437 val_437 2008-12-31 12 +469 val_469 2008-12-31 12 +192 val_192 2008-12-31 12 +286 val_286 2008-12-31 12 +187 val_187 2008-12-31 12 +176 val_176 2008-12-31 12 +54 val_54 2008-12-31 12 +459 val_459 2008-12-31 12 +51 val_51 2008-12-31 12 +138 val_138 2008-12-31 12 +103 val_103 2008-12-31 12 +239 val_239 2008-12-31 12 +213 val_213 2008-12-31 12 +216 val_216 2008-12-31 12 +430 val_430 2008-12-31 12 +278 val_278 2008-12-31 12 +176 val_176 2008-12-31 12 +289 val_289 2008-12-31 12 +221 val_221 2008-12-31 12 +65 val_65 2008-12-31 12 +318 val_318 2008-12-31 12 +332 val_332 2008-12-31 12 +311 val_311 2008-12-31 12 +275 val_275 2008-12-31 12 +137 val_137 2008-12-31 12 +241 val_241 2008-12-31 12 +83 val_83 2008-12-31 12 +333 val_333 2008-12-31 12 +180 val_180 2008-12-31 12 +284 val_284 2008-12-31 12 +12 val_12 2008-12-31 12 +230 val_230 2008-12-31 12 +181 val_181 2008-12-31 12 +67 val_67 2008-12-31 12 +260 val_260 2008-12-31 12 +404 val_404 2008-12-31 12 +384 val_384 2008-12-31 12 +489 val_489 2008-12-31 12 +353 val_353 2008-12-31 12 +373 val_373 2008-12-31 12 +272 val_272 2008-12-31 12 +138 val_138 2008-12-31 12 +217 val_217 2008-12-31 12 +84 val_84 2008-12-31 12 +348 val_348 2008-12-31 12 +466 val_466 2008-12-31 12 +58 val_58 2008-12-31 12 +8 val_8 2008-12-31 12 +411 val_411 2008-12-31 12 +230 val_230 2008-12-31 12 +208 val_208 2008-12-31 12 +348 val_348 2008-12-31 12 +24 val_24 2008-12-31 12 +463 val_463 2008-12-31 12 +431 val_431 2008-12-31 12 +179 val_179 2008-12-31 12 +172 val_172 2008-12-31 12 +42 val_42 2008-12-31 12 +129 val_129 2008-12-31 12 +158 val_158 2008-12-31 12 +119 val_119 2008-12-31 12 +496 val_496 2008-12-31 12 +0 val_0 2008-12-31 12 +322 val_322 2008-12-31 12 +197 val_197 2008-12-31 12 +468 val_468 2008-12-31 12 +393 val_393 2008-12-31 12 +454 val_454 2008-12-31 12 +100 val_100 2008-12-31 12 +298 val_298 2008-12-31 12 +199 val_199 2008-12-31 12 +191 val_191 2008-12-31 12 +418 val_418 2008-12-31 12 +96 val_96 2008-12-31 12 +26 val_26 2008-12-31 12 +165 val_165 2008-12-31 12 +327 val_327 2008-12-31 12 +230 val_230 2008-12-31 12 +205 val_205 2008-12-31 12 +120 val_120 2008-12-31 12 +131 val_131 2008-12-31 12 +51 val_51 2008-12-31 12 +404 val_404 2008-12-31 12 +43 val_43 2008-12-31 12 +436 val_436 2008-12-31 12 +156 val_156 2008-12-31 12 +469 val_469 2008-12-31 12 +468 val_468 2008-12-31 12 +308 val_308 2008-12-31 12 +95 val_95 2008-12-31 12 +196 val_196 2008-12-31 12 +288 val_288 2008-12-31 12 +481 val_481 2008-12-31 12 +457 val_457 2008-12-31 12 +98 val_98 2008-12-31 12 +282 val_282 2008-12-31 12 +197 val_197 2008-12-31 12 +187 val_187 2008-12-31 12 +318 val_318 2008-12-31 12 +318 val_318 2008-12-31 12 +409 val_409 2008-12-31 12 +470 val_470 2008-12-31 12 +137 val_137 2008-12-31 12 +369 val_369 2008-12-31 12 +316 val_316 2008-12-31 12 +169 val_169 2008-12-31 12 +413 val_413 2008-12-31 12 +85 val_85 2008-12-31 12 +77 val_77 2008-12-31 12 +0 val_0 2008-12-31 12 +490 val_490 2008-12-31 12 +87 val_87 2008-12-31 12 +364 val_364 2008-12-31 12 +179 val_179 2008-12-31 12 +118 val_118 2008-12-31 12 +134 val_134 2008-12-31 12 +395 val_395 2008-12-31 12 +282 val_282 2008-12-31 12 +138 val_138 2008-12-31 12 +238 val_238 2008-12-31 12 +419 val_419 2008-12-31 12 +15 val_15 2008-12-31 12 +118 val_118 2008-12-31 12 +72 val_72 2008-12-31 12 +90 val_90 2008-12-31 12 +307 val_307 2008-12-31 12 +19 val_19 2008-12-31 12 +435 val_435 2008-12-31 12 +10 val_10 2008-12-31 12 +277 val_277 2008-12-31 12 +273 val_273 2008-12-31 12 +306 val_306 2008-12-31 12 +224 val_224 2008-12-31 12 +309 val_309 2008-12-31 12 +389 val_389 2008-12-31 12 +327 val_327 2008-12-31 12 +242 val_242 2008-12-31 12 +369 val_369 2008-12-31 12 +392 val_392 2008-12-31 12 +272 val_272 2008-12-31 12 +331 val_331 2008-12-31 12 +401 val_401 2008-12-31 12 +242 val_242 2008-12-31 12 +452 val_452 2008-12-31 12 +177 val_177 2008-12-31 12 +226 val_226 2008-12-31 12 +5 val_5 2008-12-31 12 +497 val_497 2008-12-31 12 +402 val_402 2008-12-31 12 +396 val_396 2008-12-31 12 +317 val_317 2008-12-31 12 +395 val_395 2008-12-31 12 +58 val_58 2008-12-31 12 +35 val_35 2008-12-31 12 +336 val_336 2008-12-31 12 +95 val_95 2008-12-31 12 +11 val_11 2008-12-31 12 +168 val_168 2008-12-31 12 +34 val_34 2008-12-31 12 +229 val_229 2008-12-31 12 +233 val_233 2008-12-31 12 +143 val_143 2008-12-31 12 +472 val_472 2008-12-31 12 +322 val_322 2008-12-31 12 +498 val_498 2008-12-31 12 +160 val_160 2008-12-31 12 +195 val_195 2008-12-31 12 +42 val_42 2008-12-31 12 +321 val_321 2008-12-31 12 +430 val_430 2008-12-31 12 +119 val_119 2008-12-31 12 +489 val_489 2008-12-31 12 +458 val_458 2008-12-31 12 +78 val_78 2008-12-31 12 +76 val_76 2008-12-31 12 +41 val_41 2008-12-31 12 +223 val_223 2008-12-31 12 +492 val_492 2008-12-31 12 +149 val_149 2008-12-31 12 +449 val_449 2008-12-31 12 +218 val_218 2008-12-31 12 +228 val_228 2008-12-31 12 +138 val_138 2008-12-31 12 +453 val_453 2008-12-31 12 +30 val_30 2008-12-31 12 +209 val_209 2008-12-31 12 +64 val_64 2008-12-31 12 +468 val_468 2008-12-31 12 +76 val_76 2008-12-31 12 +74 val_74 2008-12-31 12 +342 val_342 2008-12-31 12 +69 val_69 2008-12-31 12 +230 val_230 2008-12-31 12 +33 val_33 2008-12-31 12 +368 val_368 2008-12-31 12 +103 val_103 2008-12-31 12 +296 val_296 2008-12-31 12 +113 val_113 2008-12-31 12 +216 val_216 2008-12-31 12 +367 val_367 2008-12-31 12 +344 val_344 2008-12-31 12 +167 val_167 2008-12-31 12 +274 val_274 2008-12-31 12 +219 val_219 2008-12-31 12 +239 val_239 2008-12-31 12 +485 val_485 2008-12-31 12 +116 val_116 2008-12-31 12 +223 val_223 2008-12-31 12 +256 val_256 2008-12-31 12 +263 val_263 2008-12-31 12 +70 val_70 2008-12-31 12 +487 val_487 2008-12-31 12 +480 val_480 2008-12-31 12 +401 val_401 2008-12-31 12 +288 val_288 2008-12-31 12 +191 val_191 2008-12-31 12 +5 val_5 2008-12-31 12 +244 val_244 2008-12-31 12 +438 val_438 2008-12-31 12 +128 val_128 2008-12-31 12 +467 val_467 2008-12-31 12 +432 val_432 2008-12-31 12 +202 val_202 2008-12-31 12 +316 val_316 2008-12-31 12 +229 val_229 2008-12-31 12 +469 val_469 2008-12-31 12 +463 val_463 2008-12-31 12 +280 val_280 2008-12-31 12 +2 val_2 2008-12-31 12 +35 val_35 2008-12-31 12 +283 val_283 2008-12-31 12 +331 val_331 2008-12-31 12 +235 val_235 2008-12-31 12 +80 val_80 2008-12-31 12 +44 val_44 2008-12-31 12 +193 val_193 2008-12-31 12 +321 val_321 2008-12-31 12 +335 val_335 2008-12-31 12 +104 val_104 2008-12-31 12 +466 val_466 2008-12-31 12 +366 val_366 2008-12-31 12 +175 val_175 2008-12-31 12 +403 val_403 2008-12-31 12 +483 val_483 2008-12-31 12 +53 val_53 2008-12-31 12 +105 val_105 2008-12-31 12 +257 val_257 2008-12-31 12 +406 val_406 2008-12-31 12 +409 val_409 2008-12-31 12 +190 val_190 2008-12-31 12 +406 val_406 2008-12-31 12 +401 val_401 2008-12-31 12 +114 val_114 2008-12-31 12 +258 val_258 2008-12-31 12 +90 val_90 2008-12-31 12 +203 val_203 2008-12-31 12 +262 val_262 2008-12-31 12 +348 val_348 2008-12-31 12 +424 val_424 2008-12-31 12 +12 val_12 2008-12-31 12 +396 val_396 2008-12-31 12 +201 val_201 2008-12-31 12 +217 val_217 2008-12-31 12 +164 val_164 2008-12-31 12 +431 val_431 2008-12-31 12 +454 val_454 2008-12-31 12 +478 val_478 2008-12-31 12 +298 val_298 2008-12-31 12 +125 val_125 2008-12-31 12 +431 val_431 2008-12-31 12 +164 val_164 2008-12-31 12 +424 val_424 2008-12-31 12 +187 val_187 2008-12-31 12 +382 val_382 2008-12-31 12 +5 val_5 2008-12-31 12 +70 val_70 2008-12-31 12 +397 val_397 2008-12-31 12 +480 val_480 2008-12-31 12 +291 val_291 2008-12-31 12 +24 val_24 2008-12-31 12 +351 val_351 2008-12-31 12 +255 val_255 2008-12-31 12 +104 val_104 2008-12-31 12 +70 val_70 2008-12-31 12 +163 val_163 2008-12-31 12 +438 val_438 2008-12-31 12 +119 val_119 2008-12-31 12 +414 val_414 2008-12-31 12 +200 val_200 2008-12-31 12 +491 val_491 2008-12-31 12 +237 val_237 2008-12-31 12 +439 val_439 2008-12-31 12 +360 val_360 2008-12-31 12 +248 val_248 2008-12-31 12 +479 val_479 2008-12-31 12 +305 val_305 2008-12-31 12 +417 val_417 2008-12-31 12 +199 val_199 2008-12-31 12 +444 val_444 2008-12-31 12 +120 val_120 2008-12-31 12 +429 val_429 2008-12-31 12 +169 val_169 2008-12-31 12 +443 val_443 2008-12-31 12 +323 val_323 2008-12-31 12 +325 val_325 2008-12-31 12 +277 val_277 2008-12-31 12 +230 val_230 2008-12-31 12 +478 val_478 2008-12-31 12 +178 val_178 2008-12-31 12 +468 val_468 2008-12-31 12 +310 val_310 2008-12-31 12 +317 val_317 2008-12-31 12 +333 val_333 2008-12-31 12 +493 val_493 2008-12-31 12 +460 val_460 2008-12-31 12 +207 val_207 2008-12-31 12 +249 val_249 2008-12-31 12 +265 val_265 2008-12-31 12 +480 val_480 2008-12-31 12 +83 val_83 2008-12-31 12 +136 val_136 2008-12-31 12 +353 val_353 2008-12-31 12 +172 val_172 2008-12-31 12 +214 val_214 2008-12-31 12 +462 val_462 2008-12-31 12 +233 val_233 2008-12-31 12 +406 val_406 2008-12-31 12 +133 val_133 2008-12-31 12 +175 val_175 2008-12-31 12 +189 val_189 2008-12-31 12 +454 val_454 2008-12-31 12 +375 val_375 2008-12-31 12 +401 val_401 2008-12-31 12 +421 val_421 2008-12-31 12 +407 val_407 2008-12-31 12 +384 val_384 2008-12-31 12 +256 val_256 2008-12-31 12 +26 val_26 2008-12-31 12 +134 val_134 2008-12-31 12 +67 val_67 2008-12-31 12 +384 val_384 2008-12-31 12 +379 val_379 2008-12-31 12 +18 val_18 2008-12-31 12 +462 val_462 2008-12-31 12 +492 val_492 2008-12-31 12 +100 val_100 2008-12-31 12 +298 val_298 2008-12-31 12 +9 val_9 2008-12-31 12 +341 val_341 2008-12-31 12 +498 val_498 2008-12-31 12 +146 val_146 2008-12-31 12 +458 val_458 2008-12-31 12 +362 val_362 2008-12-31 12 +186 val_186 2008-12-31 12 +285 val_285 2008-12-31 12 +348 val_348 2008-12-31 12 +167 val_167 2008-12-31 12 +18 val_18 2008-12-31 12 +273 val_273 2008-12-31 12 +183 val_183 2008-12-31 12 +281 val_281 2008-12-31 12 +344 val_344 2008-12-31 12 +97 val_97 2008-12-31 12 +469 val_469 2008-12-31 12 +315 val_315 2008-12-31 12 +84 val_84 2008-12-31 12 +28 val_28 2008-12-31 12 +37 val_37 2008-12-31 12 +448 val_448 2008-12-31 12 +152 val_152 2008-12-31 12 +348 val_348 2008-12-31 12 +307 val_307 2008-12-31 12 +194 val_194 2008-12-31 12 +414 val_414 2008-12-31 12 +477 val_477 2008-12-31 12 +222 val_222 2008-12-31 12 +126 val_126 2008-12-31 12 +90 val_90 2008-12-31 12 +169 val_169 2008-12-31 12 +403 val_403 2008-12-31 12 +400 val_400 2008-12-31 12 +200 val_200 2008-12-31 12 +97 val_97 2008-12-31 12 +PREHOOK: query: describe extended nzhang_part1 partition(ds='2008-04-08',hr=11) +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe extended nzhang_part1 partition(ds='2008-04-08',hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +key string default +value string default +ds string +hr string + +Detailed Partition Information Partition(values:[2008-04-08, 11], dbName:default, tableName:nzhang_part1, createTime:1281384647, lastAccessTime:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:default), FieldSchema(name:value, type:string, comment:default)], location:pfile:/data/users/aaly/work/apache-hive/build/ql/test/data/warehouse/nzhang_part1/ds=2008-04-08/hr=11, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), parameters:{numFiles=1, transient_lastDdlTime=1281384649, numRows=500, totalSize=5812}) +PREHOOK: query: describe extended nzhang_part1 partition(ds='2008-04-08',hr=12) +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe extended nzhang_part1 partition(ds='2008-04-08',hr=12) +POSTHOOK: type: DESCTABLE +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +key string default +value string default +ds string +hr string + +Detailed Partition Information Partition(values:[2008-04-08, 12], dbName:default, tableName:nzhang_part1, createTime:1281384648, lastAccessTime:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:default), FieldSchema(name:value, type:string, comment:default)], location:pfile:/data/users/aaly/work/apache-hive/build/ql/test/data/warehouse/nzhang_part1/ds=2008-04-08/hr=12, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), parameters:{numFiles=1, transient_lastDdlTime=1281384649, numRows=500, totalSize=5812}) +PREHOOK: query: describe extended nzhang_part2 partition(ds='2008-12-31',hr=11) +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe extended nzhang_part2 partition(ds='2008-12-31',hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +key string default +value string default +ds string +hr string + +Detailed Partition Information Partition(values:[2008-12-31, 11], dbName:default, tableName:nzhang_part2, createTime:1281384648, lastAccessTime:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:default), FieldSchema(name:value, type:string, comment:default)], location:pfile:/data/users/aaly/work/apache-hive/build/ql/test/data/warehouse/nzhang_part2/ds=2008-12-31/hr=11, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), parameters:{numFiles=1, transient_lastDdlTime=1281384649, numRows=500, totalSize=5812}) +PREHOOK: query: describe extended nzhang_part2 partition(ds='2008-12-31',hr=12) +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe extended nzhang_part2 partition(ds='2008-12-31',hr=12) +POSTHOOK: type: DESCTABLE +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +key string default +value string default +ds string +hr string + +Detailed Partition Information Partition(values:[2008-12-31, 12], dbName:default, tableName:nzhang_part2, createTime:1281384648, lastAccessTime:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:default), FieldSchema(name:value, type:string, comment:default)], location:pfile:/data/users/aaly/work/apache-hive/build/ql/test/data/warehouse/nzhang_part2/ds=2008-12-31/hr=12, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), parameters:{numFiles=1, transient_lastDdlTime=1281384649, numRows=500, totalSize=5812}) +PREHOOK: query: describe extended nzhang_part1 +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe extended nzhang_part1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +key string default +value string default +ds string +hr string + +Detailed Table Information Table(tableName:nzhang_part1, dbName:default, owner:null, createTime:1281384642, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:default), FieldSchema(name:value, type:string, comment:default)], location:pfile:/data/users/aaly/work/apache-hive/build/ql/test/data/warehouse/nzhang_part1, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[FieldSchema(name:ds, type:string, comment:null), FieldSchema(name:hr, type:string, comment:null)], parameters:{numPartitions=2, EXTERNAL=FALSE, numFiles=2, transient_lastDdlTime=1281384649, numRows=1000, totalSize=11624}, viewOriginalText:null, viewExpandedText:null, tableType:MANAGED_TABLE) +PREHOOK: query: describe extended nzhang_part2 +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe extended nzhang_part2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +key string default +value string default +ds string +hr string + +Detailed Table Information Table(tableName:nzhang_part2, dbName:default, owner:null, createTime:1281384642, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:default), FieldSchema(name:value, type:string, comment:default)], location:pfile:/data/users/aaly/work/apache-hive/build/ql/test/data/warehouse/nzhang_part2, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[FieldSchema(name:ds, type:string, comment:null), FieldSchema(name:hr, type:string, comment:null)], parameters:{numPartitions=2, EXTERNAL=FALSE, numFiles=2, transient_lastDdlTime=1281384649, numRows=1000, totalSize=11624}, viewOriginalText:null, viewExpandedText:null, tableType:MANAGED_TABLE) +PREHOOK: query: drop table nzhang_part1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table nzhang_part1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: default@nzhang_part1 +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +PREHOOK: query: drop table nzhang_part2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table nzhang_part2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: default@nzhang_part2 +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] Index: ql/src/test/results/clientpositive/stats5.q.out =================================================================== --- ql/src/test/results/clientpositive/stats5.q.out (revision 0) +++ ql/src/test/results/clientpositive/stats5.q.out (revision 0) @@ -0,0 +1,37 @@ +PREHOOK: query: explain analyze table src compute statistics +PREHOOK: type: null +POSTHOOK: query: explain analyze table src compute statistics +POSTHOOK: type: null +ABSTRACT SYNTAX TREE: + (TOK_ANALYZE (TOK_TABTYPE src)) + +STAGE DEPENDENCIES: + Stage-0 is a root stage + Stage-1 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-0 + Map Reduce + Alias -> Map Operator Tree: + src + TableScan + alias: src + + Stage: Stage-1 + Stats Operator + + +PREHOOK: query: analyze table src compute statistics +PREHOOK: type: null +PREHOOK: Input: default@src +POSTHOOK: query: analyze table src compute statistics +POSTHOOK: type: null +POSTHOOK: Input: default@src +PREHOOK: query: describe extended src +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe extended src +POSTHOOK: type: DESCTABLE +key string default +value string default + +Detailed Table Information Table(tableName:src, dbName:default, owner:null, createTime:1281386650, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:default), FieldSchema(name:value, type:string, comment:default)], location:pfile:/data/users/aaly/work/apache-hive/build/ql/test/data/warehouse/src, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[], parameters:{numPartitions=0, numFiles=1, transient_lastDdlTime=1281386656, numRows=500, totalSize=5812}, viewOriginalText:null, viewExpandedText:null, tableType:MANAGED_TABLE) Index: ql/src/test/results/clientpositive/stats6.q.out =================================================================== --- ql/src/test/results/clientpositive/stats6.q.out (revision 0) +++ ql/src/test/results/clientpositive/stats6.q.out (revision 0) @@ -0,0 +1,16 @@ +PREHOOK: query: analyze table srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics +PREHOOK: type: null +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: analyze table srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics +POSTHOOK: type: null +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: query: describe extended srcpart PARTITION(ds='2008-04-08',hr=11) +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe extended srcpart PARTITION(ds='2008-04-08',hr=11) +POSTHOOK: type: DESCTABLE +key string default +value string default +ds string +hr string + +Detailed Partition Information Partition(values:[2008-04-08, 11], dbName:default, tableName:srcpart, createTime:1280871222, lastAccessTime:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:default), FieldSchema(name:value, type:string, comment:default)], location:pfile:/data/users/aaly/work/apache-hive/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), parameters:{numFiles=1, transient_lastDdlTime=1280871231, numRows=500, totalSize=4096}) Index: ql/src/test/results/clientpositive/stats1.q.out =================================================================== --- ql/src/test/results/clientpositive/stats1.q.out (revision 0) +++ ql/src/test/results/clientpositive/stats1.q.out (revision 0) @@ -0,0 +1,209 @@ +PREHOOK: query: -- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink + +drop table tmptable +PREHOOK: type: DROPTABLE +POSTHOOK: query: -- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink + +drop table tmptable +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table tmptable(key string, value string) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table tmptable(key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tmptable +PREHOOK: query: explain +insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION ALL + select s2.key as key, s2.value as value from src1 s2) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION ALL + select s2.key as key, s2.value as value from src1 s2) unionsrc +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF src s1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR 'tst1' key) (TOK_SELEXPR (TOK_FUNCTION TOK_STRING (TOK_FUNCTION count 1)) value)))) (TOK_QUERY (TOK_FROM (TOK_TABREF src1 s2)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL s2) key) key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL s2) value) value))))) unionsrc)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB tmptable)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL unionsrc) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL unionsrc) value))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-4 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + null-subquery1:unionsrc-subquery1:s1 + TableScan + alias: s1 + Select Operator + Group By Operator + aggregations: + expr: count(1) + bucketGroup: false + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: bigint + Reduce Operator Tree: + Group By Operator + aggregations: + expr: count(VALUE._col0) + bucketGroup: false + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: + expr: 'tst1' + type: string + expr: UDFToString(_col0) + type: string + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + file:/tmp/aaly/hive_2010-08-09_13-32-25_295_4586645755203915221/-mr-10002 + Union + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: string + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: tmptable + file:/tmp/aaly/hive_2010-08-09_13-32-25_295_4586645755203915221/-mr-10003 + Union + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: string + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: tmptable + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: tmptable + + Stage: Stage-3 + Stats Operator + + Stage: Stage-4 + Map Reduce + Alias -> Map Operator Tree: + null-subquery2:unionsrc-subquery2:s2 + TableScan + alias: s2 + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + +PREHOOK: query: insert overwrite table tmptable +select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION ALL + select s2.key as key, s2.value as value from src1 s2) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Output: default@tmptable +POSTHOOK: query: insert overwrite table tmptable +select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION ALL + select s2.key as key, s2.value as value from src1 s2) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@tmptable +POSTHOOK: Lineage: tmptable.key EXPRESSION [(src1)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmptable.value EXPRESSION [(src)s1.null, (src1)s2.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from tmptable x sort by x.key, x.value +PREHOOK: type: QUERY +PREHOOK: Input: default@tmptable +PREHOOK: Output: file:/tmp/aaly/hive_2010-08-09_13-32-34_786_8877460730892721060/-mr-10000 +POSTHOOK: query: select * from tmptable x sort by x.key, x.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tmptable +POSTHOOK: Output: file:/tmp/aaly/hive_2010-08-09_13-32-34_786_8877460730892721060/-mr-10000 +POSTHOOK: Lineage: tmptable.key EXPRESSION [(src1)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmptable.value EXPRESSION [(src)s1.null, (src1)s2.FieldSchema(name:value, type:string, comment:default), ] + + + + + val_165 + val_193 + val_265 + val_27 + val_409 + val_484 +128 +146 val_146 +150 val_150 +213 val_213 +224 +238 val_238 +255 val_255 +273 val_273 +278 val_278 +311 val_311 +369 +401 val_401 +406 val_406 +66 val_66 +98 val_98 +tst1 500 +PREHOOK: query: drop table tmptable +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table tmptable +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: default@tmptable +POSTHOOK: Lineage: tmptable.key EXPRESSION [(src1)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmptable.value EXPRESSION [(src)s1.null, (src1)s2.FieldSchema(name:value, type:string, comment:default), ] Index: ql/src/test/results/clientpositive/stats10.q.out =================================================================== --- ql/src/test/results/clientpositive/stats10.q.out (revision 0) +++ ql/src/test/results/clientpositive/stats10.q.out (revision 0) @@ -0,0 +1,442 @@ +PREHOOK: query: CREATE TABLE bucket3_1(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE bucket3_1(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@bucket3_1 +PREHOOK: query: explain +insert overwrite table bucket3_1 partition (ds='1') +select * from src +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table bucket3_1 partition (ds='1') +select * from src +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB bucket3_1 (TOK_PARTSPEC (TOK_PARTVAL ds '1')))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + TableScan + alias: src + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + outputColumnNames: _col0, _col1 + Reduce Output Operator + sort order: + Map-reduce partition columns: + expr: UDFToInteger(_col0) + type: int + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: string + Reduce Operator Tree: + Extract + Select Operator + expressions: + expr: UDFToInteger(_col0) + type: int + expr: _col1 + type: string + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: bucket3_1 + + Stage: Stage-0 + Move Operator + tables: + partition: + ds 1 + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: bucket3_1 + + Stage: Stage-2 + Stats Operator + + +PREHOOK: query: insert overwrite table bucket3_1 partition (ds='1') +select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@bucket3_1@ds=1 +POSTHOOK: query: insert overwrite table bucket3_1 partition (ds='1') +select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@bucket3_1@ds=1 +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table bucket3_1 partition (ds='1') +select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@bucket3_1@ds=1 +POSTHOOK: query: insert overwrite table bucket3_1 partition (ds='1') +select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@bucket3_1@ds=1 +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table bucket3_1 partition (ds='2') +select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@bucket3_1@ds=2 +POSTHOOK: query: insert overwrite table bucket3_1 partition (ds='2') +select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@bucket3_1@ds=2 +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from bucket3_1 tablesample (bucket 1 out of 2) s where ds = '1' order by key +PREHOOK: type: QUERY +PREHOOK: Input: default@bucket3_1@ds=1 +PREHOOK: Output: file:/tmp/aaly/hive_2010-08-09_13-51-37_053_514290142284583234/-mr-10000 +POSTHOOK: query: select * from bucket3_1 tablesample (bucket 1 out of 2) s where ds = '1' order by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@bucket3_1@ds=1 +POSTHOOK: Output: file:/tmp/aaly/hive_2010-08-09_13-51-37_053_514290142284583234/-mr-10000 +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +0 val_0 1 +0 val_0 1 +0 val_0 1 +2 val_2 1 +4 val_4 1 +8 val_8 1 +10 val_10 1 +12 val_12 1 +12 val_12 1 +18 val_18 1 +18 val_18 1 +20 val_20 1 +24 val_24 1 +24 val_24 1 +26 val_26 1 +26 val_26 1 +28 val_28 1 +30 val_30 1 +34 val_34 1 +42 val_42 1 +42 val_42 1 +44 val_44 1 +54 val_54 1 +58 val_58 1 +58 val_58 1 +64 val_64 1 +66 val_66 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +72 val_72 1 +72 val_72 1 +74 val_74 1 +76 val_76 1 +76 val_76 1 +78 val_78 1 +80 val_80 1 +82 val_82 1 +84 val_84 1 +84 val_84 1 +86 val_86 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +92 val_92 1 +96 val_96 1 +98 val_98 1 +98 val_98 1 +100 val_100 1 +100 val_100 1 +104 val_104 1 +104 val_104 1 +114 val_114 1 +116 val_116 1 +118 val_118 1 +118 val_118 1 +120 val_120 1 +120 val_120 1 +126 val_126 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +134 val_134 1 +134 val_134 1 +136 val_136 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +146 val_146 1 +146 val_146 1 +150 val_150 1 +152 val_152 1 +152 val_152 1 +156 val_156 1 +158 val_158 1 +160 val_160 1 +162 val_162 1 +164 val_164 1 +164 val_164 1 +166 val_166 1 +168 val_168 1 +170 val_170 1 +172 val_172 1 +172 val_172 1 +174 val_174 1 +174 val_174 1 +176 val_176 1 +176 val_176 1 +178 val_178 1 +180 val_180 1 +186 val_186 1 +190 val_190 1 +192 val_192 1 +194 val_194 1 +196 val_196 1 +200 val_200 1 +200 val_200 1 +202 val_202 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +214 val_214 1 +216 val_216 1 +216 val_216 1 +218 val_218 1 +222 val_222 1 +224 val_224 1 +224 val_224 1 +226 val_226 1 +228 val_228 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +238 val_238 1 +238 val_238 1 +242 val_242 1 +242 val_242 1 +244 val_244 1 +248 val_248 1 +252 val_252 1 +256 val_256 1 +256 val_256 1 +258 val_258 1 +260 val_260 1 +262 val_262 1 +266 val_266 1 +272 val_272 1 +272 val_272 1 +274 val_274 1 +278 val_278 1 +278 val_278 1 +280 val_280 1 +280 val_280 1 +282 val_282 1 +282 val_282 1 +284 val_284 1 +286 val_286 1 +288 val_288 1 +288 val_288 1 +292 val_292 1 +296 val_296 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +302 val_302 1 +306 val_306 1 +308 val_308 1 +310 val_310 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +322 val_322 1 +322 val_322 1 +332 val_332 1 +336 val_336 1 +338 val_338 1 +342 val_342 1 +342 val_342 1 +344 val_344 1 +344 val_344 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +356 val_356 1 +360 val_360 1 +362 val_362 1 +364 val_364 1 +366 val_366 1 +368 val_368 1 +374 val_374 1 +378 val_378 1 +382 val_382 1 +382 val_382 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +386 val_386 1 +392 val_392 1 +394 val_394 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +400 val_400 1 +402 val_402 1 +404 val_404 1 +404 val_404 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +414 val_414 1 +414 val_414 1 +418 val_418 1 +424 val_424 1 +424 val_424 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +432 val_432 1 +436 val_436 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +444 val_444 1 +446 val_446 1 +448 val_448 1 +452 val_452 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +458 val_458 1 +458 val_458 1 +460 val_460 1 +462 val_462 1 +462 val_462 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +470 val_470 1 +472 val_472 1 +478 val_478 1 +478 val_478 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +482 val_482 1 +484 val_484 1 +490 val_490 1 +492 val_492 1 +492 val_492 1 +494 val_494 1 +496 val_496 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +PREHOOK: query: analyze table bucket3_1 compute statistics +PREHOOK: type: null +PREHOOK: Input: default@bucket3_1@ds=1 +PREHOOK: Input: default@bucket3_1@ds=2 +POSTHOOK: query: analyze table bucket3_1 compute statistics +POSTHOOK: type: null +POSTHOOK: Input: default@bucket3_1@ds=1 +POSTHOOK: Input: default@bucket3_1@ds=2 +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe extended bucket3_1 +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe extended bucket3_1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +key int +value string +ds string + +Detailed Table Information Table(tableName:bucket3_1, dbName:default, owner:aaly, createTime:1281387084, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:int, comment:null), FieldSchema(name:value, type:string, comment:null)], location:pfile:/data/users/aaly/work/apache-hive/build/ql/test/data/warehouse/bucket3_1, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:2, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[key], sortCols:[], parameters:{}), partitionKeys:[FieldSchema(name:ds, type:string, comment:null)], parameters:{numPartitions=2, numFiles=4, transient_lastDdlTime=1281387103, numRows=1000, totalSize=11624}, viewOriginalText:null, viewExpandedText:null, tableType:MANAGED_TABLE) +PREHOOK: query: describe extended bucket3_1 partition (ds='1') +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe extended bucket3_1 partition (ds='1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +key int +value string +ds string + +Detailed Partition Information Partition(values:[1], dbName:default, tableName:bucket3_1, createTime:1281387088, lastAccessTime:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:int, comment:null), FieldSchema(name:value, type:string, comment:null)], location:pfile:/data/users/aaly/work/apache-hive/build/ql/test/data/warehouse/bucket3_1/ds=1, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:2, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[key], sortCols:[], parameters:{}), parameters:{numFiles=2, transient_lastDdlTime=1281387103, numRows=500, totalSize=5812}) +PREHOOK: query: describe extended bucket3_1 partition (ds='2') +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe extended bucket3_1 partition (ds='2') +POSTHOOK: type: DESCTABLE +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +key int +value string +ds string + +Detailed Partition Information Partition(values:[2], dbName:default, tableName:bucket3_1, createTime:1281387096, lastAccessTime:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:int, comment:null), FieldSchema(name:value, type:string, comment:null)], location:pfile:/data/users/aaly/work/apache-hive/build/ql/test/data/warehouse/bucket3_1/ds=2, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:2, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[key], sortCols:[], parameters:{}), parameters:{numFiles=2, transient_lastDdlTime=1281387103, numRows=500, totalSize=5812}) Index: ql/src/test/results/clientpositive/stats7.q.out =================================================================== --- ql/src/test/results/clientpositive/stats7.q.out (revision 0) +++ ql/src/test/results/clientpositive/stats7.q.out (revision 0) @@ -0,0 +1,28 @@ +PREHOOK: query: analyze table srcpart PARTITION(ds='2008-04-08',hr) compute statistics +PREHOOK: type: null +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: analyze table srcpart PARTITION(ds='2008-04-08',hr) compute statistics +POSTHOOK: type: null +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: query: describe extended srcpart PARTITION(ds='2008-04-08',hr=11) +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe extended srcpart PARTITION(ds='2008-04-08',hr=11) +POSTHOOK: type: DESCTABLE +key string default +value string default +ds string +hr string + +Detailed Partition Information Partition(values:[2008-04-08, 11], dbName:default, tableName:srcpart, createTime:1280871291, lastAccessTime:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:default), FieldSchema(name:value, type:string, comment:default)], location:pfile:/data/users/aaly/work/apache-hive/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), parameters:{numFiles=1, transient_lastDdlTime=1280871301, numRows=500, totalSize=4096}) +PREHOOK: query: describe extended srcpart PARTITION(ds='2008-04-08',hr=12) +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe extended srcpart PARTITION(ds='2008-04-08',hr=12) +POSTHOOK: type: DESCTABLE +key string default +value string default +ds string +hr string + +Detailed Partition Information Partition(values:[2008-04-08, 12], dbName:default, tableName:srcpart, createTime:1280871292, lastAccessTime:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:default), FieldSchema(name:value, type:string, comment:default)], location:pfile:/data/users/aaly/work/apache-hive/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), parameters:{numFiles=1, transient_lastDdlTime=1280871301, numRows=500, totalSize=4096}) Index: ql/src/test/results/clientpositive/stats2.q.out =================================================================== --- ql/src/test/results/clientpositive/stats2.q.out (revision 0) +++ ql/src/test/results/clientpositive/stats2.q.out (revision 0) @@ -0,0 +1,12 @@ +PREHOOK: query: analyze table srcpart compute statistics +PREHOOK: type: null +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: query: analyze table srcpart compute statistics +POSTHOOK: type: null +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 Index: ql/src/test/queries/clientpositive/stats8.q =================================================================== --- ql/src/test/queries/clientpositive/stats8.q (revision 0) +++ ql/src/test/queries/clientpositive/stats8.q (revision 0) @@ -0,0 +1,6 @@ +analyze table srcpart PARTITION(ds, hr) compute statistics; + +describe extended srcpart PARTITION(ds='2008-04-08',hr=11); +describe extended srcpart PARTITION(ds='2008-04-09',hr=12); +describe extended srcpart PARTITION(ds='2008-04-08',hr=11); +describe extended srcpart PARTITION(ds='2008-04-09',hr=12); \ No newline at end of file Index: ql/src/test/queries/clientpositive/stats9.q =================================================================== --- ql/src/test/queries/clientpositive/stats9.q (revision 0) +++ ql/src/test/queries/clientpositive/stats9.q (revision 0) @@ -0,0 +1,5 @@ +analyze table srcbucket compute statistics; +describe extended srcbucket; + +analyze table srcbucket2 compute statistics; +describe extended srcbucket2; \ No newline at end of file Index: ql/src/test/queries/clientpositive/stats1.q =================================================================== --- ql/src/test/queries/clientpositive/stats1.q (revision 0) +++ ql/src/test/queries/clientpositive/stats1.q (revision 0) @@ -0,0 +1,24 @@ + +set hive.merge.mapfiles=false; +set hive.merge.mapredfiles=false; +set hive.map.aggr = true; + +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink + +drop table tmptable; +create table tmptable(key string, value string); + +explain +insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION ALL + select s2.key as key, s2.value as value from src1 s2) unionsrc; + +insert overwrite table tmptable +select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION ALL + select s2.key as key, s2.value as value from src1 s2) unionsrc; + +select * from tmptable x sort by x.key, x.value; + +drop table tmptable; \ No newline at end of file Index: ql/src/test/queries/clientpositive/stats10.q =================================================================== --- ql/src/test/queries/clientpositive/stats10.q (revision 0) +++ ql/src/test/queries/clientpositive/stats10.q (revision 0) @@ -0,0 +1,26 @@ +set hive.enforce.bucketing = true; +set hive.exec.reducers.max = 1; + +CREATE TABLE bucket3_1(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS; + +explain +insert overwrite table bucket3_1 partition (ds='1') +select * from src; + +insert overwrite table bucket3_1 partition (ds='1') +select * from src; + +insert overwrite table bucket3_1 partition (ds='1') +select * from src; + +insert overwrite table bucket3_1 partition (ds='2') +select * from src; + +select * from bucket3_1 tablesample (bucket 1 out of 2) s where ds = '1' order by key; + +analyze table bucket3_1 compute statistics; + + +describe extended bucket3_1; +describe extended bucket3_1 partition (ds='1'); +describe extended bucket3_1 partition (ds='2'); \ No newline at end of file Index: ql/src/test/queries/clientpositive/stats2.q =================================================================== --- ql/src/test/queries/clientpositive/stats2.q (revision 0) +++ ql/src/test/queries/clientpositive/stats2.q (revision 0) @@ -0,0 +1,3 @@ + + +analyze table srcpart compute statistics; Index: ql/src/test/queries/clientpositive/stats3.q =================================================================== --- ql/src/test/queries/clientpositive/stats3.q (revision 0) +++ ql/src/test/queries/clientpositive/stats3.q (revision 0) @@ -0,0 +1,23 @@ +drop table hive_test_src; +drop table hive_test_dst; + +create table hive_test_src ( col1 string ) stored as textfile ; +load data local inpath '../data/files/test.dat' overwrite into table hive_test_src ; + +create table hive_test_dst ( col1 string ) partitioned by ( pcol1 string , pcol2 string) stored as sequencefile; +insert overwrite table hive_test_dst partition ( pcol1='test_part', pCol2='test_Part') select col1 from hive_test_src ; +select * from hive_test_dst where pcol1='test_part' and pcol2='test_Part'; + +select count(1) from hive_test_dst; + +insert overwrite table hive_test_dst partition ( pCol1='test_part', pcol2='test_Part') select col1 from hive_test_src ; +select * from hive_test_dst where pcol1='test_part' and pcol2='test_part'; + +select count(1) from hive_test_dst; + +select * from hive_test_dst where pcol1='test_part'; +select * from hive_test_dst where pcol1='test_part' and pcol2='test_part'; +select * from hive_test_dst where pcol1='test_Part'; + +drop table hive_test_src; +drop table hive_test_dst; \ No newline at end of file Index: ql/src/test/queries/clientpositive/stats4.q =================================================================== --- ql/src/test/queries/clientpositive/stats4.q (revision 0) +++ ql/src/test/queries/clientpositive/stats4.q (revision 0) @@ -0,0 +1,38 @@ +show partitions srcpart; + +drop table nzhang_part1; +drop table nzhang_part2; + +create table if not exists nzhang_part1 like srcpart; +create table if not exists nzhang_part2 like srcpart; + +set hive.exec.dynamic.partition.mode=nonstrict; +set hive.exec.dynamic.partition=true; +set hive.stats.autogather=true; + +explain +from srcpart +insert overwrite table nzhang_part1 partition (ds, hr) select key, value, ds, hr where ds <= '2008-04-08' +insert overwrite table nzhang_part2 partition(ds='2008-12-31', hr) select key, value, hr where ds > '2008-04-08'; + +from srcpart +insert overwrite table nzhang_part1 partition (ds, hr) select key, value, ds, hr where ds <= '2008-04-08' +insert overwrite table nzhang_part2 partition(ds='2008-12-31', hr) select key, value, hr where ds > '2008-04-08'; + + +show partitions nzhang_part1; +show partitions nzhang_part2; + +select * from nzhang_part1 where ds is not null and hr is not null; +select * from nzhang_part2 where ds is not null and hr is not null; + +describe extended nzhang_part1 partition(ds='2008-04-08',hr=11); +describe extended nzhang_part1 partition(ds='2008-04-08',hr=12); +describe extended nzhang_part2 partition(ds='2008-12-31',hr=11); +describe extended nzhang_part2 partition(ds='2008-12-31',hr=12); + +describe extended nzhang_part1; +describe extended nzhang_part2; + +drop table nzhang_part1; +drop table nzhang_part2; \ No newline at end of file Index: ql/src/test/queries/clientpositive/stats5.q =================================================================== --- ql/src/test/queries/clientpositive/stats5.q (revision 0) +++ ql/src/test/queries/clientpositive/stats5.q (revision 0) @@ -0,0 +1,7 @@ + + +explain analyze table src compute statistics; + +analyze table src compute statistics; + +describe extended src; \ No newline at end of file Index: ql/src/test/queries/clientpositive/stats6.q =================================================================== --- ql/src/test/queries/clientpositive/stats6.q (revision 0) +++ ql/src/test/queries/clientpositive/stats6.q (revision 0) @@ -0,0 +1,3 @@ +analyze table srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics; + +describe extended srcpart PARTITION(ds='2008-04-08',hr=11); \ No newline at end of file Index: ql/src/test/queries/clientpositive/stats7.q =================================================================== --- ql/src/test/queries/clientpositive/stats7.q (revision 0) +++ ql/src/test/queries/clientpositive/stats7.q (revision 0) @@ -0,0 +1,4 @@ +analyze table srcpart PARTITION(ds='2008-04-08',hr) compute statistics; + +describe extended srcpart PARTITION(ds='2008-04-08',hr=11); +describe extended srcpart PARTITION(ds='2008-04-08',hr=12); \ No newline at end of file Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java (revision 982011) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java (working copy) @@ -32,6 +32,8 @@ import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.MapredWork; +import org.apache.hadoop.hive.ql.plan.StatsWork; /** * Processor for the rule - table scan. @@ -54,12 +56,12 @@ GenMRProcContext ctx = (GenMRProcContext) opProcCtx; ParseContext parseCtx = ctx.getParseCtx(); Map, GenMapRedCtx> mapCurrCtx = ctx - .getMapCurrCtx(); + .getMapCurrCtx(); // create a dummy task Task currTask = TaskFactory.get(GenMapRedUtils.getMapRedWork(parseCtx.getConf()), - parseCtx.getConf()); + parseCtx.getConf()); Operator currTopOp = op; ctx.setCurrTask(currTask); ctx.setCurrTopOp(currTopOp); @@ -70,6 +72,14 @@ String currAliasId = alias; ctx.setCurrAliasId(currAliasId); mapCurrCtx.put(op, new GenMapRedCtx(currTask, currTopOp, currAliasId)); + + if (parseCtx.getQB().getParseInfo().isAnalyzeCommand()) { + StatsWork statsWork = new StatsWork(parseCtx.getQB().getParseInfo().getTableSpec()); + Task statsTask = TaskFactory.get(statsWork, parseCtx.getConf()); + currTask.addDependentTask(statsTask); + ctx.getRootTasks().add(currTask); + GenMapRedUtils.setTaskPlan(currAliasId, currTopOp, (MapredWork) currTask.getWork(), false, ctx); + } return null; } } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java (revision 982011) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java (working copy) @@ -38,7 +38,6 @@ import org.apache.hadoop.hive.ql.exec.RowSchema; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; -import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.exec.UnionOperator; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.lib.Node; @@ -51,6 +50,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory; import org.apache.hadoop.hive.ql.plan.ConditionalResolverMergeFiles; +import org.apache.hadoop.hive.ql.plan.ConditionalResolverMergeFiles.ConditionalResolverMergeFilesCtx; import org.apache.hadoop.hive.ql.plan.ConditionalWork; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -63,9 +63,9 @@ import org.apache.hadoop.hive.ql.plan.PartitionDesc; import org.apache.hadoop.hive.ql.plan.PlanUtils; import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; +import org.apache.hadoop.hive.ql.plan.StatsWork; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.plan.TableScanDesc; -import org.apache.hadoop.hive.ql.plan.ConditionalResolverMergeFiles.ConditionalResolverMergeFilesCtx; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; /** @@ -104,16 +104,28 @@ // no need of merging if the move is to a local file system MoveTask mvTask = (MoveTask) findMoveTask(mvTasks, (FileSinkOperator) nd); + + // Add the stats task as a dependent task of the move task + if (((FileSinkOperator) nd).getConf().getTableInfo().getTableName() != null && ctx.getParseCtx().getQB().getParseInfo().isInsertToTable() /*&& !fsOp.getConf().isGatherStats()*/) { + System.out.println("AHMEDDDDDDDDD process MRFSK1"); + ((FileSinkOperator) nd).getConf().setGatherStats(true); + MoveWork mvWork = ((MoveTask)mvTask).getWork(); + StatsWork statsWork = new StatsWork(mvWork.getLoadTableWork()); + Task statsTask = TaskFactory.get(statsWork, ctx.getParseCtx().getConf()); + mvTask.addDependentTask(statsTask); + } + + if ((mvTask != null) && !mvTask.isLocal()) { // There are separate configuration parameters to control whether to // merge for a map-only job // or for a map-reduce job if ((parseCtx.getConf().getBoolVar( HiveConf.ConfVars.HIVEMERGEMAPFILES) && (((MapredWork) currTask - .getWork()).getReducer() == null)) - || (parseCtx.getConf().getBoolVar( - HiveConf.ConfVars.HIVEMERGEMAPREDFILES) && (((MapredWork) currTask - .getWork()).getReducer() != null))) { + .getWork()).getReducer() == null)) + || (parseCtx.getConf().getBoolVar( + HiveConf.ConfVars.HIVEMERGEMAPREDFILES) && (((MapredWork) currTask + .getWork()).getReducer() != null))) { chDir = true; } } @@ -169,7 +181,7 @@ // Add the extract operator to get the value fields RowResolver out_rwsch = new RowResolver(); RowResolver interim_rwsch = ctx.getParseCtx().getOpParseCtx().get(fsOp) - .getRR(); + .getRR(); Integer pos = Integer.valueOf(0); for (ColumnInfo colInfo : interim_rwsch.getColumnInfos()) { String[] info = interim_rwsch.reverseLookup(colInfo.getInternalName()); @@ -180,17 +192,17 @@ Operator extract = OperatorFactory.getAndMakeChild(new ExtractDesc( new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, - Utilities.ReduceField.VALUE.toString(), "", false)), - new RowSchema(out_rwsch.getColumnInfos())); + Utilities.ReduceField.VALUE.toString(), "", false)), + new RowSchema(out_rwsch.getColumnInfos())); TableDesc ts = (TableDesc) fsConf.getTableInfo().clone(); fsConf - .getTableInfo() - .getProperties() - .remove( + .getTableInfo() + .getProperties() + .remove( org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_PARTITION_COLUMNS); FileSinkOperator newOutput = (FileSinkOperator) OperatorFactory - .getAndMakeChild(new FileSinkDesc(finalName, ts, parseCtx.getConf() + .getAndMakeChild(new FileSinkDesc(finalName, ts, parseCtx.getConf() .getBoolVar(HiveConf.ConfVars.COMPRESSRESULT)), fsRS, extract); cplan.setReducer(extract); @@ -301,7 +313,7 @@ Operator currTopOp = ctx.getCurrTopOp(); String currAliasId = ctx.getCurrAliasId(); HashMap, Task> opTaskMap = ctx - .getOpTaskMap(); + .getOpTaskMap(); List> seenOps = ctx.getSeenOps(); List> rootTasks = ctx.getRootTasks(); @@ -332,7 +344,7 @@ // mapTask and currTask should be merged by and join/union operator // (e.g., GenMRUnion1j) which has multiple topOps. assert mapTask == currTask : "mapTask.id = " + mapTask.getId() - + "; currTask.id = " + currTask.getId(); + + "; currTask.id = " + currTask.getId(); } return dest; Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (revision 982011) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (working copy) @@ -59,16 +59,16 @@ import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.FetchWork; import org.apache.hadoop.hive.ql.plan.FileSinkDesc; +import org.apache.hadoop.hive.ql.plan.FilterDesc.sampleDesc; import org.apache.hadoop.hive.ql.plan.MapJoinDesc; import org.apache.hadoop.hive.ql.plan.MapredLocalWork; +import org.apache.hadoop.hive.ql.plan.MapredLocalWork.BucketMapJoinContext; import org.apache.hadoop.hive.ql.plan.MapredWork; import org.apache.hadoop.hive.ql.plan.PartitionDesc; import org.apache.hadoop.hive.ql.plan.PlanUtils; import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.plan.TableScanDesc; -import org.apache.hadoop.hive.ql.plan.FilterDesc.sampleDesc; -import org.apache.hadoop.hive.ql.plan.MapredLocalWork.BucketMapJoinContext; /** * General utility common functions for the Processor to convert operator into @@ -90,15 +90,15 @@ * processing context */ public static void initPlan(ReduceSinkOperator op, GenMRProcContext opProcCtx) - throws SemanticException { + throws SemanticException { Operator reducer = op.getChildOperators().get(0); Map, GenMapRedCtx> mapCurrCtx = opProcCtx - .getMapCurrCtx(); + .getMapCurrCtx(); GenMapRedCtx mapredCtx = mapCurrCtx.get(op.getParentOperators().get(0)); Task currTask = mapredCtx.getCurrTask(); MapredWork plan = (MapredWork) currTask.getWork(); HashMap, Task> opTaskMap = opProcCtx - .getOpTaskMap(); + .getOpTaskMap(); Operator currTopOp = opProcCtx.getCurrTopOp(); opTaskMap.put(reducer, currTask); @@ -148,9 +148,9 @@ public static void initMapJoinPlan(Operator op, GenMRProcContext opProcCtx, boolean readInputMapJoin, boolean readInputUnion, boolean setReducer, int pos, boolean createLocalPlan) - throws SemanticException { + throws SemanticException { Map, GenMapRedCtx> mapCurrCtx = opProcCtx - .getMapCurrCtx(); + .getMapCurrCtx(); assert (((pos == -1) && (readInputMapJoin)) || (pos != -1)); int parentPos = (pos == -1) ? 0 : pos; GenMapRedCtx mapredCtx = mapCurrCtx.get(op.getParentOperators().get( @@ -158,7 +158,7 @@ Task currTask = mapredCtx.getCurrTask(); MapredWork plan = (MapredWork) currTask.getWork(); HashMap, Task> opTaskMap = opProcCtx - .getOpTaskMap(); + .getOpTaskMap(); Operator currTopOp = opProcCtx.getCurrTopOp(); // The mapjoin has already been encountered. Some context must be stored @@ -171,7 +171,7 @@ if (setReducer) { Operator reducer = op.getChildOperators() - .get(0); + .get(0); plan.setReducer(reducer); opTaskMap.put(reducer, currTask); if (reducer.getClass() == JoinOperator.class) { @@ -292,12 +292,12 @@ GenMRProcContext opProcCtx) throws SemanticException { Operator reducer = op.getChildOperators().get(0); Map, GenMapRedCtx> mapCurrCtx = opProcCtx - .getMapCurrCtx(); + .getMapCurrCtx(); GenMapRedCtx mapredCtx = mapCurrCtx.get(op.getParentOperators().get(0)); Task currTask = mapredCtx.getCurrTask(); MapredWork plan = (MapredWork) currTask.getWork(); HashMap, Task> opTaskMap = opProcCtx - .getOpTaskMap(); + .getOpTaskMap(); opTaskMap.put(reducer, currTask); plan.setReducer(reducer); @@ -469,7 +469,7 @@ * processing context */ public static void splitPlan(ReduceSinkOperator op, GenMRProcContext opProcCtx) - throws SemanticException { + throws SemanticException { // Generate a new task ParseContext parseCtx = opProcCtx.getParseCtx(); MapredWork cplan = getMapRedWork(parseCtx.getConf()); @@ -484,7 +484,7 @@ cplan.setNumReduceTasks(new Integer(desc.getNumReducers())); HashMap, Task> opTaskMap = opProcCtx - .getOpTaskMap(); + .getOpTaskMap(); opTaskMap.put(reducer, redTask); Task currTask = opProcCtx.getCurrTask(); @@ -558,6 +558,14 @@ for (Partition part : parts) { if (part.getTable().isPartitioned()) { + if (parseCtx.getQB().getParseInfo().isAnalyzeCommand()) { + if (parseCtx.getQB().getParseInfo().getTableSpec().partSpec != null) { + if (!comparePartSpecs(parseCtx.getQB().getParseInfo().getTableSpec().partSpec, part.getSpec())) { + continue; + } + } + } + inputs.add(new ReadEntity(part)); } else { inputs.add(new ReadEntity(part.getTable())); @@ -587,8 +595,9 @@ continue; } String path = p.toString(); - if (LOG.isDebugEnabled()) + if (LOG.isDebugEnabled()) { LOG.debug("Adding " + path + " of table" + alias_id); + } partDir.add(p); try { @@ -616,8 +625,9 @@ } plan.getPathToAliases().get(path).add(alias_id); plan.getPathToPartitionInfo().put(path, prtDesc); - if (LOG.isDebugEnabled()) + if (LOG.isDebugEnabled()) { LOG.debug("Information added for path " + path); + } } assert plan.getAliasToWork().get(alias_id) == null; @@ -636,10 +646,10 @@ localPlan.getAliasToWork().put(alias_id, topOp); if (tblDir == null) { localPlan.getAliasToFetchWork() - .put( + .put( alias_id, new FetchWork(FetchWork.convertPathToStringArray(partDir), - partDesc)); + partDesc)); } else { localPlan.getAliasToFetchWork().put(alias_id, new FetchWork(tblDir.toString(), tblDesc)); @@ -719,7 +729,7 @@ tagToSchema.set(tag, rs.getConf().getValueSerializeInfo()); } else { List> children = topOp - .getChildOperators(); + .getChildOperators(); if (children != null) { for (Operator op : children) { setKeyAndValueDesc(plan, op); @@ -741,7 +751,7 @@ work.setTagToValueDesc(new ArrayList()); work.setReducer(null); work.setHadoopSupportsSplittable( - conf.getBoolVar(HiveConf.ConfVars.HIVE_COMBINE_INPUT_FORMAT_SUPPORTS_SPLITTABLE)); + conf.getBoolVar(HiveConf.ConfVars.HIVE_COMBINE_INPUT_FORMAT_SUPPORTS_SPLITTABLE)); return work; } @@ -814,7 +824,7 @@ // replace the reduce child with this operator List> childOpList = parent - .getChildOperators(); + .getChildOperators(); for (int pos = 0; pos < childOpList.size(); pos++) { if (childOpList.get(pos) == op) { childOpList.set(pos, fs_op); @@ -823,7 +833,7 @@ } List> parentOpList = - new ArrayList>(); + new ArrayList>(); parentOpList.add(parent); fs_op.setParentOperators(parentOpList); @@ -837,7 +847,7 @@ op.getParentOperators().set(posn, ts_op); Map, GenMapRedCtx> mapCurrCtx = opProcCtx - .getMapCurrCtx(); + .getMapCurrCtx(); mapCurrCtx.put(ts_op, new GenMapRedCtx(childTask, null, null)); String streamDesc = taskTmpDir; @@ -962,4 +972,32 @@ // prevent instantiation } + private static boolean comparePartSpecs(Map partA, LinkedHashMappartB) { + + if (partA.size() != partB.size()) { + return false; + } + Iterator iterA = partA.keySet().iterator(); + Iterator iterB = partB.keySet().iterator(); + + while (iterA.hasNext()) { + String keyA = iterA.next(); + String keyB = iterB.next(); + + if (!keyA.equals(keyB)) { + LOG.error("Error, Invalid partition column"); + return false; + } + + if (partA.get(keyA) == null) { + continue; + } + + if (!partA.get(keyA).equals(partB.get(keyB))) { + return false; + } + } + + return true; + } } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java (revision 982011) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java (working copy) @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.exec; +import java.io.File; import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; @@ -32,6 +33,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.FileUtils; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.JobCloseFeedBack.FeedBackType; import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; @@ -45,26 +47,28 @@ import org.apache.hadoop.hive.ql.plan.PlanUtils; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.plan.api.OperatorType; +import org.apache.hadoop.hive.ql.stats.StatsFactory; +import org.apache.hadoop.hive.ql.stats.StatsPublisher; +import org.apache.hadoop.hive.ql.stats.StatsSetupConst; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.Serializer; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.SubStructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.util.ReflectionUtils; - /** * File Sink operator implementation. **/ public class FileSinkOperator extends TerminalOperator implements - Serializable { +Serializable { protected transient HashMap valToPaths; protected transient int numDynParts; @@ -100,6 +104,7 @@ Path[] outPaths; Path[] finalPaths; RecordWriter[] outWriters; + Stat stat; public FSPaths() { } @@ -109,6 +114,7 @@ outPaths = new Path[numFiles]; finalPaths = new Path[numFiles]; outWriters = new RecordWriter[numFiles]; + stat = new Stat(); } /** @@ -169,7 +175,7 @@ throw new HiveException(e); } } - } + } } private void commit(FileSystem fs) throws HiveException { @@ -192,10 +198,10 @@ if (outWriters[idx] != null) { try { outWriters[idx].close(abort); - if (delete) { - fs.delete(outPaths[idx], true); - } - updateProgress(); + if (delete) { + fs.delete(outPaths[idx], true); + } + updateProgress(); } catch (IOException e) { throw new HiveException(e); } @@ -261,11 +267,11 @@ /* String specPath; Path tmpPath; - */ + */ String taskId; private boolean filesCreated = false; - @Override + @Override protected void initializeOp(Configuration hconf) throws HiveException { try { this.hconf = hconf; @@ -324,12 +330,11 @@ if (!bDynParts) { fsp = new FSPaths(specPath); - // Create all the files - this is required because empty files need to be created for - // empty buckets - // createBucketFiles(fsp); - valToPaths.put("", fsp); // special entry for non-DP case + // Create all the files - this is required because empty files need to be created for + // empty buckets + // createBucketFiles(fsp); + valToPaths.put("", fsp); // special entry for non-DP case } - initializeChildren(hconf); } catch (HiveException e) { throw e; @@ -428,9 +433,9 @@ } try { // The reason to keep these instead of using - // OutputFormat.getRecordWriter() is that - // getRecordWriter does not give us enough control over the file name that - // we create. + // OutputFormat.getRecordWriter() is that + // getRecordWriter does not give us enough control over the file name that + // we create. if (!bDynParts) { fsp.finalPaths[filesIdx] = HiveFileFormatUtils.getOutputFormatFinalPath( parent, taskId, jc, hiveOutputFormat, isCompressed, fsp.finalPaths[filesIdx]); @@ -448,21 +453,21 @@ } LOG.info("New Final Path: FS " + fsp.finalPaths[filesIdx]); - if (isNativeTable) { - try { - // in recent hadoop versions, use deleteOnExit to clean tmp files. - autoDelete = ShimLoader.getHadoopShims().fileSystemDeleteOnExit( - fs, fsp.outPaths[filesIdx]); - } catch (IOException e) { - throw new HiveException(e); - } - } + if (isNativeTable) { + try { + // in recent hadoop versions, use deleteOnExit to clean tmp files. + autoDelete = ShimLoader.getHadoopShims().fileSystemDeleteOnExit( + fs, fsp.outPaths[filesIdx]); + } catch (IOException e) { + throw new HiveException(e); + } + } Utilities.copyTableJobPropertiesToConf(conf.getTableInfo(), jc); // only create bucket files only if no dynamic partitions, // buckets of dynamic partitions will be created for each newly created partition fsp.outWriters[filesIdx] = HiveFileFormatUtils.getHiveRecordWriter( - jc, conf.getTableInfo(), outputClass, conf, fsp.outPaths[filesIdx]); + jc, conf.getTableInfo(), outputClass, conf, fsp.outPaths[filesIdx]); filesIdx++; } assert filesIdx == numFiles; @@ -545,6 +550,11 @@ recordValue = serializer.serialize(row, subSetOI); } else { rowOutWriters = fsp.outWriters; + + if (conf.isGatherStats()) { + fsp.stat.increaseNumRows(1); + } + // use SerDe to serialize r, and write it out recordValue = serializer.serialize(row, inputObjInspectors[0]); } @@ -561,7 +571,7 @@ for (int i = 0; i < partitionEval.length; i++) { Object o = partitionEval[i].evaluate(row); keyHashCode = keyHashCode * 31 - + ObjectInspectorUtils.hashCode(o, partitionObjectInspectors[i]); + + ObjectInspectorUtils.hashCode(o, partitionObjectInspectors[i]); } key.setHashCode(keyHashCode); int bucketNum = prtner.getBucket(key, null, totalFiles); @@ -584,6 +594,7 @@ if (dpDir != null) { FSPaths fsp2 = valToPaths.get(dpDir); + if (fsp2 == null) { // check # of dp if (valToPaths.size() > maxPartitions) { @@ -597,6 +608,9 @@ createBucketFiles(fsp2); valToPaths.put(dpDir, fsp2); } + if (conf.isGatherStats()) { + fsp2.stat.increaseNumRows(1); + } rw = fsp2.outWriters; } else { rw = fsp.outWriters; @@ -617,11 +631,17 @@ errMsg.append("Operator ").append(getOperatorId()).append(" (id=").append(id).append("): "); errMsg.append(counterCode > FATAL_ERR_MSG.length - 1 ? "fatal error": - FATAL_ERR_MSG[(int) counterCode]); + FATAL_ERR_MSG[(int) counterCode]); } @Override public void closeOp(boolean abort) throws HiveException { + + // Only publish stats if this operator's flag was set to gather stats (in the semantic analyzer). + if (conf.isGatherStats()) { + publishStats(); + } + if (!bDynParts && !filesCreated) { createBucketFiles(fsp); } @@ -654,7 +674,7 @@ @Override public void jobClose(Configuration hconf, boolean success, JobCloseFeedBack feedBack) - throws HiveException { + throws HiveException { try { if ((conf != null) && isNativeTable) { String specPath = conf.getDirName(); @@ -711,7 +731,7 @@ * @throws IOException */ private void createEmptyBuckets(Configuration hconf, ArrayList paths) - throws HiveException, IOException { + throws HiveException, IOException { JobConf jc; if (hconf instanceof JobConf) { @@ -726,9 +746,9 @@ TableDesc tableInfo = conf.getTableInfo(); try { Serializer serializer = (Serializer) tableInfo.getDeserializerClass().newInstance(); - serializer.initialize(null, tableInfo.getProperties()); - outputClass = serializer.getSerializedClass(); - hiveOutputFormat = conf.getTableInfo().getOutputFileFormatClass().newInstance(); + serializer.initialize(null, tableInfo.getProperties()); + outputClass = serializer.getSerializedClass(); + hiveOutputFormat = conf.getTableInfo().getOutputFileFormatClass().newInstance(); } catch (SerDeException e) { throw new HiveException(e); } catch (InstantiationException e) { @@ -754,6 +774,30 @@ @Override public void augmentPlan() { PlanUtils.configureTableJobPropertiesForStorageHandler( - getConf().getTableInfo()); + getConf().getTableInfo()); } + + private void publishStats() { + // Initializing a stats publisher + StatsPublisher statsPublisher; + String statsImplementationClass = HiveConf.getVar(hconf, HiveConf.ConfVars.HIVESTATSDBCLASS); + if (StatsFactory.setImplementation(statsImplementationClass, jc)) { + statsPublisher = StatsFactory.getStatsPublisher(); + statsPublisher.connect(hconf); + + for (String fspKey : valToPaths.keySet()) { + FSPaths fspValue = valToPaths.get(fspKey); + String prefix = conf.getStaticSpec(); + if (fspKey != "") { + prefix += File.separator; + } + /* "prefix + fspKey" forms the key of the temporary storage relation. This key + is the table name in case of non-partitioned tables, but it is + "TableName + PartitionSpec" in case of partitions */ + statsPublisher.publishStat(prefix + fspKey, StatsSetupConst.ROW_COUNT, Long.toString(fspValue.stat.getNumRows())); + } + + statsPublisher.closeConnection(); + } + } } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java (revision 982011) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java (working copy) @@ -31,6 +31,7 @@ import org.apache.hadoop.hive.ql.plan.FunctionWork; import org.apache.hadoop.hive.ql.plan.MapredWork; import org.apache.hadoop.hive.ql.plan.MoveWork; +import org.apache.hadoop.hive.ql.plan.StatsWork; /** * TaskFactory implementation. @@ -67,6 +68,8 @@ ConditionalTask.class)); taskvec.add(new taskTuple(MapredWork.class, MapRedTask.class)); + taskvec.add(new taskTuple(StatsWork.class, + StatsTask.class)); } private static ThreadLocal tid = new ThreadLocal() { @@ -130,7 +133,7 @@ } makeChild(ret, tasklist); - + return (ret); } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java (revision 982011) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java (working copy) @@ -19,10 +19,25 @@ package org.apache.hadoop.hive.ql.exec; import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.FileUtils; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.ql.plan.api.OperatorType; +import org.apache.hadoop.hive.ql.stats.StatsFactory; +import org.apache.hadoop.hive.ql.stats.StatsPublisher; +import org.apache.hadoop.hive.ql.stats.StatsSetupConst; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.mapred.JobConf; /** * Table Scan Operator If the data is coming from the map-reduce framework, just @@ -30,25 +45,112 @@ * read as part of map-reduce framework **/ public class TableScanOperator extends Operator implements - Serializable { +Serializable { private static final long serialVersionUID = 1L; + private transient List writable; + private transient List values; + private transient Configuration hconf; + private transient String defaultPartitionName; + private transient int dpStartCol; + private transient String partitionName; + private transient Stat stat; + protected transient JobConf jc; + /** - * Currently, the table scan operator does not do anything special other than - * just forwarding the row. Since the table data is always read as part of the - * map-reduce framework by the mapper. But, this assumption is not true, i.e - * table data is not only read by the mapper, this operator will be enhanced - * to read the table. + * Other than gathering statistics for the ANALYZE command, the table scan operator + * does not do anything special other than just forwarding the row. Since the table + * data is always read as part of the map-reduce framework by the mapper. But, this + * assumption is not true, i.e table data is not only read by the mapper, this + * operator will be enhanced to read the table. **/ @Override public void processOp(Object row, int tag) throws HiveException { + if (conf != null) { + if (conf.isGatherStats()) { + if (stat == null) { + stat = new Stat(); + if (conf.getPartColumns().size() == 0) { + partitionName = ""; + } + else { + assert inputObjInspectors[0].getCategory() == ObjectInspector.Category.STRUCT: + "input object inspector is not struct"; + + writable.clear(); + values.clear(); + ObjectInspectorUtils.partialCopyToStandardObject(writable, row, dpStartCol, conf.getPartColumns().size(), + (StructObjectInspector) inputObjInspectors[0], ObjectInspectorCopyOption.WRITABLE); + + for (Object o: writable) { + if (o == null || o.toString().length() == 0) { + values.add(defaultPartitionName); + } else { + values.add(o.toString()); + } + } + partitionName = FileUtils.makePartName(conf.getPartColumns(), values); + } + } + stat.increaseNumRows(1); + } + } + forward(row, inputObjInspectors[tag]); } + @Override + protected void initializeOp(Configuration hconf) throws HiveException { + initializeChildren(hconf); + if (conf == null) { + return; + } + if (!conf.isGatherStats()) { + return; + } + + this.hconf = hconf; + if (hconf instanceof JobConf) { + jc = (JobConf) hconf; + } else { + // test code path + jc = new JobConf(hconf, ExecDriver.class); + } + + stat = null; + if (conf.getPartColumns().size() == 0) { + // NON PARTITIONED table + return; + } + + writable = new ArrayList(conf.getPartColumns().size()); + values = new ArrayList(conf.getPartColumns().size()); + defaultPartitionName = HiveConf.getVar(hconf, HiveConf.ConfVars.DEFAULTPARTITIONNAME); + this.dpStartCol = 0; + StructObjectInspector soi = (StructObjectInspector) inputObjInspectors[0]; + for (StructField sf: soi.getAllStructFieldRefs()) { + String fn = sf.getFieldName(); + if (!conf.getPartColumns().contains(fn)) { + this.dpStartCol++; + } else { + break; + } + } + } + + @Override + public void closeOp(boolean abort) throws HiveException { + if (conf != null) { + if (conf.isGatherStats() && stat != null) { + publishStats(); + } + } + } + /** * The operator name for this operator type. This is used to construct the * rule for an operator - * + * * @return the operator name **/ @Override @@ -74,4 +176,29 @@ public int getType() { return OperatorType.TABLESCAN; } + + private void publishStats() { + // Initializing a stats publisher + StatsPublisher statsPublisher; + String statsImplementationClass = HiveConf.getVar(hconf, HiveConf.ConfVars.HIVESTATSDBCLASS); + if (StatsFactory.setImplementation(statsImplementationClass, jc)) { + statsPublisher = StatsFactory.getStatsPublisher(); + statsPublisher.connect(hconf); + + String prefix; + prefix = conf.getAlias(); + if (partitionName.isEmpty()) { + // No partitions, key for temp storage is just the table name + statsPublisher.publishStat(conf.getAlias(), StatsSetupConst.ROW_COUNT, + Long.toString(stat.getNumRows())); + } else { + // It's a partition, key for temp storage is "tableName/partitionSpecs" + // ex: table1/ds=2008-09-09/hr=11 + statsPublisher.publishStat(conf.getAlias() + Path.SEPARATOR + partitionName, + StatsSetupConst.ROW_COUNT, Long.toString(stat.getNumRows())); + } + + statsPublisher.closeConnection(); + } + } } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java (revision 0) @@ -0,0 +1,466 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.ql.Context; +import org.apache.hadoop.hive.ql.DriverContext; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx; +import org.apache.hadoop.hive.ql.plan.LoadTableDesc; +import org.apache.hadoop.hive.ql.plan.StatsWork; +import org.apache.hadoop.hive.ql.plan.api.StageType; +import org.apache.hadoop.hive.ql.stats.StatsAggregator; +import org.apache.hadoop.hive.ql.stats.StatsFactory; +import org.apache.hadoop.hive.ql.stats.StatsSetupConst; +import org.apache.hadoop.util.StringUtils; + +/** + * StatsTask implementation. + **/ +public class StatsTask extends Task implements Serializable { + + private static final long serialVersionUID = 1L; + + public StatsTask() { + super(); + } + + @Override + public int execute(DriverContext driverContext) { + + if (work.getTableSpecs() != null) { + return aggregateStatsAnalyze(); + } else if (work.getLoadTableDesc() != null){ + return aggregateStatsInsert(); + } + else { + return 1; + } + } + + @Override + public int getType() { + return StageType.STATS; + } + + @Override + public String getName() { + return "STATS"; + } + + @Override + protected void localizeMRTmpFilesImpl(Context ctx) { + // Nothing to do for StatsTask here. + } + + private int aggregateStatsInsert() { + + try { + // Stats setup: + Warehouse wh = new Warehouse(conf); + FileSystem fileSys; + FileStatus[] fileStatus; + int nPartitionsInTable = 0, nFilesInTable = 0, nFilesInPartition; + long oldNRowsInTable = 0; int oldNFilesInTable = 0, oldSizeOfTable = 0, oldNPartitions = 0; + long partitionSize, tableSize = 0, nRowsInPartition, nRowsInTable = 0; + long oldNRowsInPartition = 0; int oldNFilesInPartition = 0, oldSizeOfPartition = 0; + + StatsAggregator statsAggregator; + String statsImplementationClass = HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS); + StatsFactory.setImplementation(statsImplementationClass, conf); + statsAggregator = StatsFactory.getStatsAggregator(); + if (!statsAggregator.connect(conf)) { + LOG.error("Failure. Could not aggregate Statistics."); + return 1; + } + + LoadTableDesc tbd = work.getLoadTableDesc(); + Table table = db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tbd + .getTable().getTableName()); + + // Stats calculations for non-partitioned tables: + if (tbd.getPartitionSpec().size() == 0) { + nPartitionsInTable = 0; + Path tablePath = wh.getDefaultTablePath(MetaStoreUtils.DEFAULT_DATABASE_NAME, tbd.getTable().getTableName()); + fileSys = tablePath.getFileSystem(conf); + fileStatus = Utilities.getFileStatusRecurse(tablePath, 1, fileSys); + nFilesInTable = fileStatus.length; + tableSize = 0; + for (int i = 0; i < fileStatus.length; i++) { + tableSize += fileStatus[i].getLen(); + } + + // In that case (No Partitions) the key for stats is the table name! + nRowsInTable = Long.parseLong(statsAggregator.aggregateStats(tbd.getTable().getTableName(), StatsSetupConst.ROW_COUNT)); + } + // Partitioned table: + else { + DynamicPartitionCtx dpCtx = tbd.getDPCtx(); + if (dpCtx != null && dpCtx.getNumDPCols() > 0) { // dynamic partitions + // load the list of DP partitions and return the list of partition specs + ArrayList> fullSpecs = getPartitionFullSpecs( + new Path(tbd.getSourceDir()), tbd.getPartitionSpec(),dpCtx.getNumDPCols()); + + nFilesInTable = 0; + tableSize = nRowsInTable = 0; + + // for each partition spec, get the partition + // and put it to WriteEntity for post-exec hook + for (LinkedHashMap partSpec: fullSpecs) { + Partition partn = db.getPartition(table, partSpec, false); + + String partitionID = tbd.getTable().getTableName(); + for (String key : partSpec.keySet()) { + partitionID += Path.SEPARATOR + key + "=" + partSpec.get(key); + } + + nRowsInPartition = Long.parseLong(statsAggregator.aggregateStats(partitionID, StatsSetupConst.ROW_COUNT)); + + fileSys = partn.getPartitionPath().getFileSystem(conf); + fileStatus = Utilities.getFileStatusRecurse(partn.getPartitionPath(), 1, fileSys); + nFilesInPartition = fileStatus.length; + + partitionSize = 0; + for (int i = 0; i < fileStatus.length; i++) { + partitionSize += fileStatus[i].getLen(); + } + + // Writing partition stats to metastore + org.apache.hadoop.hive.metastore.api.Partition tPart = partn.getTPartition(); + Map parameters = tPart.getParameters(); + + if (parameters.containsKey(StatsSetupConst.ROW_COUNT)) { + oldNRowsInPartition = Long.parseLong(parameters.get(StatsSetupConst.ROW_COUNT)); + } + else { + nPartitionsInTable++; + } + + parameters.put(StatsSetupConst.ROW_COUNT, Long.toString(nRowsInPartition)); + + if (parameters.containsKey(StatsSetupConst.NUM_FILES)) { + oldNFilesInPartition = Integer.parseInt(parameters.get(StatsSetupConst.NUM_FILES)); + } + parameters.put(StatsSetupConst.NUM_FILES, Integer.toString(nFilesInPartition)); + + if (parameters.containsKey(StatsSetupConst.TOTAL_SIZE)) { + oldSizeOfPartition = Integer.parseInt(parameters.get(StatsSetupConst.TOTAL_SIZE)); + } + parameters.put(StatsSetupConst.TOTAL_SIZE, Long.toString(partitionSize)); + + nRowsInTable += (nRowsInPartition - oldNRowsInPartition); + nFilesInTable += (nFilesInPartition - oldNFilesInPartition); + tableSize += (partitionSize - oldSizeOfPartition); + + db.alterPartition(tbd.getTable().getTableName(), new Partition(table, tPart)); + } + + } else { // static partitions + // Stats Calculations for static partitions + Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false); + String partitionID = tbd.getTable().getTableName(); + for (String key : tbd.getPartitionSpec().keySet()) { + partitionID += Path.SEPARATOR + key + "=" + tbd.getPartitionSpec().get(key); + } + + nRowsInPartition = Long.parseLong(statsAggregator.aggregateStats(partitionID, StatsSetupConst.ROW_COUNT)); + + fileSys = partn.getPartitionPath().getFileSystem(conf); + fileStatus = Utilities.getFileStatusRecurse(partn.getPartitionPath(), 1, fileSys); + nFilesInPartition = fileStatus.length; + + tableSize = 0; + partitionSize = 0; + for (int i = 0; i < fileStatus.length; i++) { + partitionSize += fileStatus[i].getLen(); + } + + // Writing partition stats to metastore + + org.apache.hadoop.hive.metastore.api.Partition tPart = partn.getTPartition(); + Map parameters = tPart.getParameters(); + + if (parameters.containsKey(StatsSetupConst.ROW_COUNT)) { + oldNRowsInPartition = Long.parseLong(parameters.get(StatsSetupConst.ROW_COUNT)); + } + else { + nPartitionsInTable++; + } + parameters.put(StatsSetupConst.ROW_COUNT, Long.toString(nRowsInPartition)); + + if (parameters.containsKey(StatsSetupConst.NUM_FILES)) { + oldNFilesInPartition = Integer.parseInt(parameters.get(StatsSetupConst.NUM_FILES)); + } + parameters.put(StatsSetupConst.NUM_FILES, Integer.toString(nFilesInPartition)); + + if (parameters.containsKey(StatsSetupConst.TOTAL_SIZE)) { + oldSizeOfPartition = Integer.parseInt(parameters.get(StatsSetupConst.TOTAL_SIZE)); + } + parameters.put(StatsSetupConst.TOTAL_SIZE, Long.toString(partitionSize)); + + nRowsInTable += (nRowsInPartition - oldNRowsInPartition); + nFilesInTable += (nFilesInPartition - oldNFilesInPartition); + tableSize += (partitionSize - oldSizeOfPartition); + + db.alterPartition(tbd.getTable().getTableName(), new Partition(table, tPart)); + } + + + } + + // Writing table stats to metastore + org.apache.hadoop.hive.metastore.api.Table tTable = table.getTTable(); + Map parameters = tTable.getParameters(); + if (parameters.containsKey(StatsSetupConst.ROW_COUNT) && table.isPartitioned()) { + oldNRowsInTable = Long.parseLong(parameters.get(StatsSetupConst.ROW_COUNT)); + } + nRowsInTable += oldNRowsInTable; + parameters.put(StatsSetupConst.ROW_COUNT, Long.toString(nRowsInTable)); + + if (parameters.containsKey(StatsSetupConst.NUM_PARTITIONS) && table.isPartitioned()) { + oldNPartitions = Integer.parseInt(parameters.get(StatsSetupConst.NUM_PARTITIONS)); + } + nPartitionsInTable += oldNPartitions; + parameters.put(StatsSetupConst.NUM_PARTITIONS, Integer.toString(nPartitionsInTable)); + + if (parameters.containsKey(StatsSetupConst.NUM_FILES) && table.isPartitioned()) { + oldNFilesInTable = Integer.parseInt(parameters.get(StatsSetupConst.NUM_FILES)); + } + nFilesInTable += oldNFilesInTable; + parameters.put(StatsSetupConst.NUM_FILES, Integer.toString(nFilesInTable)); + + if (parameters.containsKey(StatsSetupConst.TOTAL_SIZE) && table.isPartitioned()) { + oldSizeOfTable = Integer.parseInt(parameters.get(StatsSetupConst.TOTAL_SIZE)); + } + tableSize += oldSizeOfTable; + parameters.put(StatsSetupConst.TOTAL_SIZE, Long.toString(tableSize)); + + LOG.info("Stats For Table: " + tbd.getTable().getTableName()); + LOG.info("Number of partitions is " + nPartitionsInTable); + LOG.info("Number of files is " + nFilesInTable); + LOG.info("Number of rows is " + nRowsInTable); + LOG.info("Total size is " + tableSize); + + System.out.println("Stats For Table: " + tbd.getTable().getTableName()); + System.out.println("Number of partitions is " + nPartitionsInTable); + System.out.println("Number of files is " + nFilesInTable); + System.out.println("Number of rows is " + nRowsInTable); + System.out.println("Total size is " + tableSize); + + console.printInfo("Number of rows is " + nRowsInTable); + + statsAggregator.closeConnection(); + + db.alterTable(tbd.getTable().getTableName(), new Table(tTable)); + + return 0; + } + catch (Exception e) { + console.printError("Failed with exception " + e.getMessage(), "\n" + + StringUtils.stringifyException(e)); + return (1); + } + + } + + private int aggregateStatsAnalyze() { + try { + StatsAggregator statsAggregator; + String statsImplementationClass = HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS); + StatsFactory.setImplementation(statsImplementationClass, conf); + statsAggregator = StatsFactory.getStatsAggregator(); + + if (!statsAggregator.connect(conf)) { + LOG.error("Failure. Could not aggregate Statistics."); + return 1; + } + + Warehouse wh = new Warehouse(conf); + FileSystem fileSys; + FileStatus[] fileStatus = null; + int nPartitionsInTable = 0, nFilesInTable = 0, nFilesInPartition; + long partitionSize, tableSize = 0, nRowsInPartition, nRowsInTable = 0; + + Table table = db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, work.getTableSpecs().tableName); + if (!table.isPartitioned()) { + // Non-Partitioned tables + nPartitionsInTable = 0; + Path tablePath = wh.getDefaultTablePath(MetaStoreUtils.DEFAULT_DATABASE_NAME, work.getTableSpecs().tableName); + fileSys = tablePath.getFileSystem(conf); + fileStatus = Utilities.getFileStatusRecurse(tablePath, 1, fileSys); + nFilesInTable = fileStatus.length; + tableSize = 0; + for (int i = 0; i < fileStatus.length; i++) { + tableSize += fileStatus[i].getLen(); + } + + // In that case (No Partitions) the key for stats is the table name! + nRowsInTable = Long.parseLong(statsAggregator.aggregateStats(work.getTableSpecs().tableName, StatsSetupConst.ROW_COUNT)); + } + else { + List partitions = db.getPartitions(table); + nPartitionsInTable = partitions.size(); + nFilesInTable = 0; + tableSize = nRowsInTable = 0; + for (Partition part : partitions) { + if (work.getTableSpecs().partSpec != null) { + if (!comparePartSpecs(work.getTableSpecs().partSpec, part.getSpec())) { + continue; + } + } + + String partitionID = work.getTableSpecs().tableName; + for (String key : part.getSpec().keySet()) { + partitionID += Path.SEPARATOR + key + "=" + part.getSpec().get(key); + } + + // Reading and Writing partition stats from/to Hive MetaStore + org.apache.hadoop.hive.metastore.api.Partition tPart = part.getTPartition(); + Map parameters = tPart.getParameters(); + nRowsInPartition = Long.parseLong(statsAggregator.aggregateStats(partitionID, StatsSetupConst.ROW_COUNT)); + parameters.put(StatsSetupConst.ROW_COUNT, Long.toString(nRowsInPartition)); + + fileSys = part.getPartitionPath().getFileSystem(conf); + fileStatus = Utilities.getFileStatusRecurse(part.getPartitionPath(), 1, fileSys); + nFilesInPartition = fileStatus.length; + parameters.put(StatsSetupConst.NUM_FILES, Integer.toString(nFilesInPartition)); + + partitionSize = 0; + for (int i = 0; i < fileStatus.length; i++) { + partitionSize += fileStatus[i].getLen(); + } + parameters.put(StatsSetupConst.TOTAL_SIZE, Long.toString(partitionSize)); + + nRowsInTable += nRowsInPartition; + nFilesInTable += nFilesInPartition; + tableSize += partitionSize; + + db.alterPartition(work.getTableSpecs().tableName, new Partition(table, tPart)); + } + } + + // Writing table stats to metastore + if (work.getTableSpecs().partSpec == null) { + org.apache.hadoop.hive.metastore.api.Table tTable = table.getTTable(); + Map parameters = tTable.getParameters(); + parameters.put(StatsSetupConst.ROW_COUNT, Long.toString(nRowsInTable)); + parameters.put(StatsSetupConst.NUM_PARTITIONS, Integer.toString(nPartitionsInTable)); + parameters.put(StatsSetupConst.NUM_FILES, Integer.toString(nFilesInTable)); + parameters.put(StatsSetupConst.TOTAL_SIZE, Long.toString(tableSize)); + + LOG.info("Stats For Table: " + work.getTableSpecs().tableName); + LOG.info("Number of partitions is " + nPartitionsInTable); + LOG.info("Number of files is " + nFilesInTable); + LOG.info("Number of rows is " + nRowsInTable); + LOG.info("Total size is " + tableSize); + + System.out.println("Stats For Table: " + work.getTableSpecs().tableName); + System.out.println("Number of partitions is " + nPartitionsInTable); + System.out.println("Number of files is " + nFilesInTable); + System.out.println("Number of rows is " + nRowsInTable); + System.out.println("Total size is " + tableSize); + + db.alterTable(work.getTableSpecs().tableName, new Table(tTable)); + } + + statsAggregator.closeConnection(); + return 0; + } catch (Exception e) { + LOG.error("Error. An exception was thrown during stats aggregations." + e); + e.printStackTrace(); + return 1; + } + } + + private boolean comparePartSpecs(Map partA, LinkedHashMappartB) { + + if (partA.size() != partB.size()) { + return false; + } + Iterator iterA = partA.keySet().iterator(); + Iterator iterB = partB.keySet().iterator(); + + while (iterA.hasNext()) { + String keyA = iterA.next(); + String keyB = iterB.next(); + + if (!keyA.equals(keyB)) { + LOG.error("Error, Invalid partition column"); + return false; + } + + if (partA.get(keyA) == null) { + continue; + } + + if (!partA.get(keyA).equals(partB.get(keyB))) { + return false; + } + } + + return true; + } + + private ArrayList> getPartitionFullSpecs(Path loadPath, + Map partSpec, int numDP) + throws HiveException { + try { + ArrayList> fullPartSpecs = + new ArrayList>(); + + FileSystem fs = loadPath.getFileSystem(conf); + FileStatus[] status = Utilities.getFileStatusRecurse(loadPath, numDP, fs); + + + // for each dynamically created DP directory, construct a full partition spec + // and load the partition based on that + for (int i= 0; i < status.length; ++i) { + // get the dynamically created directory + Path partPath = status[i].getPath(); + assert fs.getFileStatus(partPath).isDir(): + "partitions " + partPath + " is not a directory !"; + + // generate a full partition specification + LinkedHashMap fullPartSpec = new LinkedHashMap(partSpec); + Warehouse.makeSpecFromName(fullPartSpec, partPath); + fullPartSpecs.add(fullPartSpec); + } + return fullPartSpecs; + } catch (IOException e) { + throw new HiveException(e); + } + + } +} \ No newline at end of file Index: ql/src/java/org/apache/hadoop/hive/ql/exec/Stat.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/Stat.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Stat.java (revision 0) @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec; + +public class Stat { + + private long numRows; + + public Stat () { + numRows = 0; + } + + public void increaseNumRows(long amount) { + numRows += amount; + } + + public long getNumRows() { + return numRows; + } + +} Index: ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java (revision 0) @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.plan; + +import java.io.Serializable; + +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec; + +/** + * ConditionalStats. + * + */ +@Explain(displayName = "Stats Operator") +public class StatsWork implements Serializable { + private static final long serialVersionUID = 1L; + + private tableSpec tableSpecs; + private LoadTableDesc loadTableDesc; + + public StatsWork() { + } + + public StatsWork(tableSpec tableSpecs) { + this.tableSpecs = tableSpecs; + } + + public StatsWork(LoadTableDesc loadTableDesc) { + this.loadTableDesc = loadTableDesc; + } + + public tableSpec getTableSpecs() { + return tableSpecs; + } + + public LoadTableDesc getLoadTableDesc() { + return loadTableDesc; + } + +} Index: ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java (revision 982011) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java (working copy) @@ -39,6 +39,8 @@ private ArrayList partitionCols; private int numFiles; private DynamicPartitionCtx dpCtx; + private String staticSpec; + private boolean gatherStats; public FileSinkDesc() { } @@ -190,4 +192,22 @@ public DynamicPartitionCtx getDynPartCtx() { return this.dpCtx; } + + public void setStaticSpec(String staticSpec) { + this.staticSpec = staticSpec; + } + + public String getStaticSpec() { + return staticSpec; + } + + public void setGatherStats(boolean gatherStats) { + this.gatherStats = gatherStats; + } + + @Explain(displayName = "GatherStats", normalExplain = false) + public boolean isGatherStats() { + return gatherStats; + } + } Index: ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java (revision 982011) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java (working copy) @@ -33,9 +33,21 @@ private static final long serialVersionUID = 1L; private String alias; - + private List virtualCols; + /** + * A list of the partition columns of the table. + * Set by the semantic analyzer only in case of the analyze command. + */ + private List partColumns; + + /** + * A boolean variable set to true by the semantic analyzer only in case of the analyze command. + * + */ + private boolean gatherStats; + @SuppressWarnings("nls") public TableScanDesc() { } @@ -43,7 +55,7 @@ public TableScanDesc(final String alias) { this.alias = alias; } - + public TableScanDesc(final String alias, List vcs) { this.alias = alias; this.virtualCols = vcs; @@ -58,6 +70,23 @@ this.alias = alias; } + public void setPartColumns (List partColumns) { + this.partColumns = partColumns; + } + + public List getPartColumns () { + return partColumns; + } + + public void setGatherStats(boolean gatherStats) { + this.gatherStats = gatherStats; + } + + @Explain(displayName = "GatherStats", normalExplain = false) + public boolean isGatherStats() { + return gatherStats; + } + public List getVirtualCols() { return virtualCols; } @@ -65,5 +94,4 @@ public void setVirtualCols(List virtualCols) { this.virtualCols = virtualCols; } - } Index: ql/src/java/org/apache/hadoop/hive/ql/stats/StatsAggregator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/stats/StatsAggregator.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/stats/StatsAggregator.java (revision 0) @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.stats; + +import org.apache.hadoop.conf.Configuration; + +/** + * An interface for any possible implementation for gathering statistics. + */ + +public interface StatsAggregator { + + /** + * This method connects to the temporary storage. + */ + public boolean connect(Configuration hconf); + + /** + * This method aggregates a given statistic from a disk storage. + * After aggregation, this method does cleaning by removing all records from the disk storage that have the same given rowID. + * + * rowID : a string identification the statistic to be gathered, possibly the table name + the partition specs. + * + * key : a string noting the key to be gathered. Ex: "numRows". + * + * */ + public String aggregateStats(String rowID, String key); + + /** + * This method closes the connection to the temporary storage. + */ + public boolean closeConnection(); + +} Index: ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java (revision 0) @@ -0,0 +1,147 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.stats.jdbc; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.stats.StatsPublisher; +import org.apache.hadoop.hive.ql.stats.StatsSetupConst; + +public class JDBCStatsPublisher implements StatsPublisher { + + private Connection conn; + private String connectionString; + private Configuration hiveconf; + private final Log LOG = LogFactory.getLog(this.getClass().getName()); + + public boolean connect(Configuration hiveconf) { + try { + this.hiveconf = hiveconf; + connectionString = HiveConf.getVar(hiveconf, HiveConf.ConfVars.HIVESTATSDBCONNECTIONSTRING); + String driver = HiveConf.getVar(hiveconf, HiveConf.ConfVars.HIVESTATSJDBCDRIVER); + Class.forName(driver).newInstance(); + conn = DriverManager.getConnection(connectionString); + } catch (Exception e) { + LOG.error("Error during JDBC initialization. " + e); + return false; + } + return true; + } + + public boolean publishStat(String rowID, String key, String value) { + + System.out.println("-------------------------------------------------------------------------------------"); + System.out.println("--------------------------- STATS PUBLISHER ------------------------------"); + System.out.println("-------------------------------------START-------------------------------------------"); + System.out.println(); + System.out.println("For KEY = " + rowID + " VALUE = " + value); + Statement stmt; + try { + stmt = conn.createStatement(); + if (key == StatsSetupConst.ROW_COUNT) { + // Appent quotes to rowID because it is a string column + rowID = "'" + rowID + "'"; + + String insert = "INSERT INTO " + JDBCStatsSetupConstants.PART_STAT_TABLE_NAME + + " VALUES (" + rowID + ", " + value + ")"; + + stmt.executeUpdate(insert); + } + else { + LOG.warn("Warning. Invalid statistic. Currently " + + "row count is the only supported statistic"); + return false; + } + stmt.close(); + } catch (SQLException e) { + LOG.error("Error during publishing statistics. " + e); + return false; + } + System.out.println(); + System.out.println("-------------------------------------------------------------------------------------"); + System.out.println("--------------------------- STATS PUBLISHER ------------------------------"); + System.out.println("----------------------------------- SUCCESS -----------------------------------------"); + return true; + } + + public boolean closeConnection() { + if (conn == null) { + return true; + } + try { + conn.close(); + + // In case of derby, explicitly close the database connection + if(HiveConf.getVar(hiveconf, HiveConf.ConfVars.HIVESTATSDBCLASS).equalsIgnoreCase("jdbc:derby")) { + try { + // The following closes the derby connection. It throws an exception that has to be caught and ignored. + DriverManager.getConnection(connectionString + ";shutdown=true"); + } + catch (Exception e) { + // Do nothing because we know that an exception is thrown anyway. + } + } + } catch (SQLException e) { + LOG.error("Error during JDBC termination. " + e); + return false; + } + + return true; + } + + public boolean init(Configuration hconf) { + try { + this.hiveconf = hconf; + connectionString = HiveConf.getVar(hconf, HiveConf.ConfVars.HIVESTATSDBCONNECTIONSTRING); + String driver = HiveConf.getVar(hconf, HiveConf.ConfVars.HIVESTATSJDBCDRIVER); + Class.forName(driver).newInstance(); + conn = DriverManager.getConnection(connectionString); + + Statement stmt = conn.createStatement(); + + // Check if the table exists + DatabaseMetaData dbm = conn.getMetaData(); + ResultSet rs = dbm.getTables(null, null, JDBCStatsSetupConstants.PART_STAT_TABLE_NAME, null); + if (!rs.next()) { // Table does not exist, create it + String createTable = "CREATE TABLE " + JDBCStatsSetupConstants.PART_STAT_TABLE_NAME + " (" + + JDBCStatsSetupConstants.PART_STAT_ID_COLUMN_NAME + " VARCHAR(255), " + + JDBCStatsSetupConstants.PART_STAT_ROW_COUNT_COLUMN_NAME + " BIGINT)"; + + stmt.executeUpdate(createTable); + stmt.close(); + } + + closeConnection(); + } catch (Exception e) { + LOG.error("Error during JDBC initialization. " + e); + return false; + } + return true; + } + +} Index: ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsSetupConstants.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsSetupConstants.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsSetupConstants.java (revision 0) @@ -0,0 +1,11 @@ +package org.apache.hadoop.hive.ql.stats.jdbc; + +public final class JDBCStatsSetupConstants { + + public static final String PART_STAT_TABLE_NAME = "PARTITION_STAT_TBL"; + + public static final String PART_STAT_ID_COLUMN_NAME = "ID"; + + public static final String PART_STAT_ROW_COUNT_COLUMN_NAME = "ROW_COUNT"; + +} Index: ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java (revision 0) @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.stats.jdbc; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.stats.StatsAggregator; +import org.apache.hadoop.hive.ql.stats.StatsSetupConst; + +public class JDBCStatsAggregator implements StatsAggregator { + + private Connection conn; + private String connectionString; + private Configuration hiveconf; + private final Log LOG = LogFactory.getLog(this.getClass().getName()); + + public boolean connect(Configuration hiveconf) { + try { + this.hiveconf = hiveconf; + connectionString = HiveConf.getVar(hiveconf, HiveConf.ConfVars.HIVESTATSDBCONNECTIONSTRING); + String driver = HiveConf.getVar(hiveconf, HiveConf.ConfVars.HIVESTATSJDBCDRIVER); + Class.forName(driver).newInstance(); + conn = DriverManager.getConnection(connectionString); + } catch (Exception e) { + LOG.error("Error during JDBC initialization. " + e); + return false; + } + return true; + } + + @Override + public String aggregateStats(String rowID, String key) { + System.out.println("-------------------------------------------------------------------------------------"); + System.out.println("--------------------------- STATS AGGREGATOR ------------------------------"); + System.out.println("-------------------------------------START-------------------------------------------"); + System.out.println(); + System.out.println("For KEY = " + rowID); + long retval = 0; + Statement stmt; + try { + // Appent quotes to rowID because it is a string column + rowID = "'" + rowID + "'"; + + if (key == StatsSetupConst.ROW_COUNT) { + stmt = conn.createStatement(); + String select = "SELECT SUM" + "(" + JDBCStatsSetupConstants.PART_STAT_ROW_COUNT_COLUMN_NAME + ")" + + " FROM " + JDBCStatsSetupConstants.PART_STAT_TABLE_NAME + + " WHERE " + JDBCStatsSetupConstants.PART_STAT_ID_COLUMN_NAME + " = " + rowID; + + ResultSet result = stmt.executeQuery(select); + if (result.next()) { + retval = result.getLong(1); + } else { + return null; + } + stmt.clearBatch(); + + /* Automatic Cleaning: + IMPORTANT: Since we publish and aggregate only 1 value (1 column) which is the row count, it + is valid to delete the row after aggregation (automatic cleaning) because we know that there is no + other values to aggregate. + If ;in the future; other values are aggregated and published, then we cannot do cleaning except + when we are sure that all values are aggregated, or we can separate the implementation of cleaning + through a separate method which the developer has to call it manually in the code. + */ + String delete = "DELETE FROM " + JDBCStatsSetupConstants.PART_STAT_TABLE_NAME + + " WHERE " + JDBCStatsSetupConstants.PART_STAT_ID_COLUMN_NAME + " = " + rowID; + stmt.executeUpdate(delete); + stmt.close(); + } else { + LOG.warn("Warning. Invalid statistic. Currently " + + "row count is the only supported statistic"); + return null; + } + } catch (SQLException e) { + LOG.error("Error during publishing aggregation. " + e); + return null; + } + System.out.println("Got VALUE = " + retval); + System.out.println(); + System.out.println("-------------------------------------------------------------------------------------"); + System.out.println("--------------------------- STATS AGGREGATOR -----------------------------"); + System.out.println("----------------------------------- SUCCESS -----------------------------------------"); + return Long.toString(retval); + } + + public boolean closeConnection() { + + if (conn == null) { + return true; + } + + try { + conn.close(); + // In case of derby, explicitly close the database connection + if(HiveConf.getVar(hiveconf, HiveConf.ConfVars.HIVESTATSDBCLASS).equalsIgnoreCase("jdbc:derby")) { + try { + // The following closes the derby connection. It throws an exception that has to be caught and ignored. + DriverManager.getConnection(connectionString + ";shutdown=true"); + } + catch (Exception e) { + // Do nothing because we know that an exception is thrown anyway. + } + } + } catch (SQLException e) { + LOG.error("Error during JDBC termination. " + e); + return false; + } + return true; + } + +} Index: ql/src/java/org/apache/hadoop/hive/ql/stats/StatsSetupConst.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/stats/StatsSetupConst.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/stats/StatsSetupConst.java (revision 0) @@ -0,0 +1,39 @@ +package org.apache.hadoop.hive.ql.stats; + +/** + * A class that defines the constant strings used by the statistics implementation. + */ + +public class StatsSetupConst { + + /** + * The value of the user variable "hive.stats.dbclass" to use HBase implementation. + */ + public static final String HBASE_IMPL_CLASS_VAL = "hbase"; + + /** + * The value of the user variable "hive.stats.dbclass" to use JDBC implementation. + */ + public static final String JDBC_IMPL_CLASS_VAL = "jdbc"; + + /** + * The name of the statistic Row Count to be published or gathered. + */ + public static final String ROW_COUNT = "numRows"; + + /** + * The name of the statistic Row Count to be published or gathered. + */ + public static final String NUM_FILES = "numFiles"; + + /** + * The name of the statistic Row Count to be published or gathered. + */ + public static final String NUM_PARTITIONS = "numPartitions"; + + /** + * The name of the statistic Row Count to be published or gathered. + */ + public static final String TOTAL_SIZE = "totalSize"; + +} Index: ql/src/java/org/apache/hadoop/hive/ql/stats/StatsFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/stats/StatsFactory.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/stats/StatsFactory.java (revision 0) @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.stats; + +import java.io.Serializable; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ReflectionUtils; + +/** + * A factory of stats publisher and aggregator implementations of the + * StatsPublisher and StatsAggregator interfaces. + */ +public final class StatsFactory { + + private static Class publisherImplementation; + private static Class aggregatorImplementation; + private static Configuration jobConf; + + /** + * Sets the paths of the implementation classes of publishing + * and aggregation (IStatsPublisher and IStatsAggregator interfaces). + * The paths are determined according to a configuration parameter which + * is passed as the user input for choosing the implementation as MySQL, HBase, ... + */ + public static boolean setImplementation(String configurationParam, Configuration conf) { + + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + // Case: hbase + if (configurationParam.equals(StatsSetupConst.HBASE_IMPL_CLASS_VAL)) { + try { + publisherImplementation = (Class) Class.forName("org.apache.hadoop.hive.hbase.HBaseStatsPublisher", true, classLoader); + aggregatorImplementation = (Class) Class.forName("org.apache.hadoop.hive.hbase.HBaseStatsAggregator", true, classLoader); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + return false; + } + } + // Case: jdbc:mysql or jdbc:derby + else if (configurationParam.contains(StatsSetupConst.JDBC_IMPL_CLASS_VAL)) { + try { + publisherImplementation = (Class) Class.forName("org.apache.hadoop.hive.ql.stats.jdbc.JDBCStatsPublisher", true, classLoader); + aggregatorImplementation = (Class) Class.forName("org.apache.hadoop.hive.ql.stats.jdbc.JDBCStatsAggregator", true, classLoader); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + return false; + } + } + else { + // ERROR + return false; + } + + jobConf = conf; + return true; + } + + /** + * Returns a Stats publisher implementation class for the IStatsPublisher interface + * For example HBaseStatsPublisher for the HBase implementation + */ + public static StatsPublisher getStatsPublisher() { + + return (StatsPublisher) ReflectionUtils.newInstance(publisherImplementation, jobConf); + } + + /** + * Returns a Stats Aggregator implementation class for the IStatsAggregator interface + * For example HBaseStatsAggregator for the HBase implementation + */ + public static StatsAggregator getStatsAggregator() { + + return (StatsAggregator) ReflectionUtils.newInstance(aggregatorImplementation, jobConf); + } + +} Index: ql/src/java/org/apache/hadoop/hive/ql/stats/StatsPublisher.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/stats/StatsPublisher.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/stats/StatsPublisher.java (revision 0) @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.stats; + +import org.apache.hadoop.conf.Configuration; + +/** + * An interface for any possible implementation for publishing statics. + */ + +public interface StatsPublisher { + + /** + * This method does the necessary one-time initializations, possibly creating the tables and schema (if do not exist). + */ + public boolean init(Configuration hconf); + + /** + * This method connects to the temporary storage. + */ + public boolean connect(Configuration hconf); + + /** + * This method publishes a given statistic into a disk storage, possibly HBase or MySQL. + * + * rowID : a string identification the statistics to be published then gathered, possibly the table name + the partition specs. + * + * key : a string noting the key to be published. Ex: "numRows". + * + * value : an integer noting the value of the published key. + * */ + public boolean publishStat(String rowID, String key, String value); + + /** + * This method closes the connection to the temporary storage. + */ + public boolean closeConnection(); + +} \ No newline at end of file Index: ql/src/java/org/apache/hadoop/hive/ql/Context.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/Context.java (revision 982011) +++ ql/src/java/org/apache/hadoop/hive/ql/Context.java (working copy) @@ -34,14 +34,14 @@ import org.antlr.runtime.TokenRewriteStream; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.conf.Configuration; /** * Context for Semantic Analyzers. Usage: not reusable - construct a new one for @@ -68,11 +68,13 @@ private final Map fsScratchDirs = new HashMap(); - private Configuration conf; + private final Configuration conf; protected int pathid = 10000; protected boolean explain = false; private TokenRewriteStream tokenRewriteStream; + private boolean gatheringStats; + String executionId; public Context(Configuration conf) throws IOException { @@ -86,17 +88,17 @@ public Context(Configuration conf, String executionId) { this.conf = conf; this.executionId = executionId; - + // non-local tmp location is configurable. however it is the same across // all external file systems nonLocalScratchPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR), - executionId); + executionId); // local tmp location is not configurable for now localScratchDir = System.getProperty("java.io.tmpdir") - + Path.SEPARATOR + System.getProperty("user.name") + Path.SEPARATOR - + executionId; + + Path.SEPARATOR + System.getProperty("user.name") + Path.SEPARATOR + + executionId; } /** @@ -106,7 +108,7 @@ public void setExplain(boolean value) { explain = value; } - + /** * Find whether the current query is an explain query * @return true if the query is an explain query, false if not @@ -119,13 +121,13 @@ /** * Get a tmp directory on specified URI * - * @param scheme Scheme of the target FS + * @param scheme Scheme of the target FS * @param authority Authority of the target FS * @param mkdir create the directory if true * @param scratchdir path of tmp directory */ private String getScratchDir(String scheme, String authority, - boolean mkdir, String scratchDir) { + boolean mkdir, String scratchDir) { String fileSystem = scheme + ":" + authority; String dir = fsScratchDirs.get(fileSystem); @@ -135,9 +137,10 @@ if (mkdir) { try { FileSystem fs = dirPath.getFileSystem(conf); - if (!fs.mkdirs(dirPath)) + if (!fs.mkdirs(dirPath)) { throw new RuntimeException("Cannot make directory: " - + dirPath.toString()); + + dirPath.toString()); + } } catch (IOException e) { throw new RuntimeException (e); } @@ -157,7 +160,7 @@ FileSystem fs = FileSystem.getLocal(conf); URI uri = fs.getUri(); return getScratchDir(uri.getScheme(), uri.getAuthority(), - mkdir, localScratchDir); + mkdir, localScratchDir); } catch (IOException e) { throw new RuntimeException (e); } @@ -166,20 +169,21 @@ /** * Create a map-reduce scratch directory on demand and return it. - * + * */ public String getMRScratchDir() { // if we are executing entirely on the client side - then // just (re)use the local scratch directory - if(isLocalOnlyExecutionMode()) + if(isLocalOnlyExecutionMode()) { return getLocalScratchDir(!explain); + } try { Path dir = FileUtils.makeQualified(nonLocalScratchPath, conf); URI uri = dir.toUri(); return getScratchDir(uri.getScheme(), uri.getAuthority(), - !explain, uri.getPath()); + !explain, uri.getPath()); } catch (IOException e) { throw new RuntimeException(e); @@ -191,7 +195,7 @@ private String getExternalScratchDir(URI extURI) { return getScratchDir(extURI.getScheme(), extURI.getAuthority(), - !explain, nonLocalScratchPath.toUri().getPath()); + !explain, nonLocalScratchPath.toUri().getPath()); } /** @@ -204,7 +208,7 @@ p.getFileSystem(conf).delete(p, true); } catch (Exception e) { LOG.warn("Error Removing Scratch: " - + StringUtils.stringifyException(e)); + + StringUtils.stringifyException(e)); } } fsScratchDirs.clear(); @@ -226,23 +230,23 @@ */ public boolean isMRTmpFileURI(String uriStr) { return (uriStr.indexOf(executionId) != -1) && - (uriStr.indexOf(MR_PREFIX) != -1); + (uriStr.indexOf(MR_PREFIX) != -1); } /** * Get a path to store map-reduce intermediate data in. - * + * * @return next available path for map-red intermediate data */ public String getMRTmpFileURI() { return getMRScratchDir() + Path.SEPARATOR + MR_PREFIX + - nextPathId(); + nextPathId(); } /** - * Given a URI for mapreduce intermediate output, swizzle the - * it to point to the local file system. This can be called in + * Given a URI for mapreduce intermediate output, swizzle the + * it to point to the local file system. This can be called in * case the caller decides to run in local mode (in which case * all intermediate data can be stored locally) * @@ -254,13 +258,14 @@ Path mrbase = new Path(getMRScratchDir()); URI relURI = mrbase.toUri().relativize(o.toUri()); - if (relURI.equals(o.toUri())) + if (relURI.equals(o.toUri())) { throw new RuntimeException - ("Invalid URI: " + originalURI + ", cannot relativize against" + - mrbase.toString()); + ("Invalid URI: " + originalURI + ", cannot relativize against" + + mrbase.toString()); + } - return getLocalScratchDir(!explain) + Path.SEPARATOR + - relURI.getPath(); + return getLocalScratchDir(!explain) + Path.SEPARATOR + + relURI.getPath(); } @@ -271,7 +276,7 @@ */ public String getLocalTmpFileURI() { return getLocalScratchDir(true) + Path.SEPARATOR + LOCAL_PREFIX + - nextPathId(); + nextPathId(); } /** @@ -283,7 +288,7 @@ */ public String getExternalTmpFileURI(URI extURI) { return getExternalScratchDir(extURI) + Path.SEPARATOR + EXT_PREFIX + - nextPathId(); + nextPathId(); } /** @@ -444,7 +449,7 @@ Random rand = new Random(); SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss_SSS"); String executionId = "hive_" + format.format(new Date()) + "_" - + Math.abs(rand.nextLong()); + + Math.abs(rand.nextLong()); return executionId; } @@ -470,14 +475,16 @@ } public void addCS(String path, ContentSummary cs) { - if(pathToCS == null) + if(pathToCS == null) { pathToCS = new HashMap (); + } pathToCS.put(path, cs); } - + public ContentSummary getCS(String path) { - if(pathToCS == null) + if(pathToCS == null) { pathToCS = new HashMap (); + } return pathToCS.get(path); } @@ -517,4 +524,12 @@ } paths.addAll(toAdd); } + + public void setGatheringStats() { + gatheringStats = true; + } + + public boolean isGatheringStats() { + return gatheringStats; + } } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java (revision 982011) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java (working copy) @@ -28,10 +28,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec; /** * Implementation of the parse information related to a query block. - * + * **/ public class QBParseInfo { @@ -46,6 +47,12 @@ private final Map destToSelExpr; private final HashMap destToWhereExpr; private final HashMap destToGroupby; + + private boolean isAnalyzeCommand; // used for the analyze command (statistics) + private boolean isInsertToTable; // used for insert overwrite command (statistics) + + private final HashMap tableSpecs; // used for statistics + /** * ClusterBy is a short name for both DistributeBy and SortBy. */ @@ -100,6 +107,8 @@ outerQueryLimit = -1; aliasToLateralViews = new HashMap>(); + + tableSpecs = new HashMap(); } public void setAggregationExprsForClause(String clause, @@ -137,7 +146,7 @@ /** * Set the Cluster By AST for the clause. - * + * * @param clause * the name of the clause * @param ast @@ -149,7 +158,7 @@ /** * Set the Distribute By AST for the clause. - * + * * @param clause * the name of the clause * @param ast @@ -161,7 +170,7 @@ /** * Set the Sort By AST for the clause. - * + * * @param clause * the name of the clause * @param ast @@ -213,7 +222,7 @@ /** * Get the Cluster By AST for the clause. - * + * * @param clause * the name of the clause * @return the abstract syntax tree @@ -228,7 +237,7 @@ /** * Get the Distribute By AST for the clause. - * + * * @param clause * the name of the clause * @return the abstract syntax tree @@ -243,7 +252,7 @@ /** * Get the Sort By AST for the clause. - * + * * @param clause * the name of the clause * @return the abstract syntax tree @@ -397,4 +406,38 @@ } lateralViews.add(lateralView); } + + public void setIsAnalyzeCommand(boolean isAnalyzeCommand) { + this.isAnalyzeCommand = isAnalyzeCommand; + } + + public boolean isAnalyzeCommand() { + return isAnalyzeCommand; + } + + public void setIsInsertToTable(boolean isInsertToTable) { + this.isInsertToTable = isInsertToTable; + } + + public boolean isInsertToTable() { + return isInsertToTable; + } + + public void addTableSpec(String tName, tableSpec tSpec) { + tableSpecs.put(tName, tSpec); + } + + public tableSpec getTableSpec(String tName) { + return tableSpecs.get(tName); + } + + /** + * This method is used only for the anlayze command to get the partition specs + */ + public tableSpec getTableSpec() { + + Iterator tName = tableSpecs.keySet().iterator(); + return tableSpecs.get(tName.next()); + } + } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (revision 982011) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (working copy) @@ -164,6 +164,7 @@ TOK_LEFTSEMIJOIN; TOK_LATERAL_VIEW; TOK_TABALIAS; +TOK_ANALYZE; } @@ -228,6 +229,7 @@ | dropIndexStatement | alterIndexRebuild | dropFunctionStatement + | analyzeStatement ; ifNotExists @@ -507,6 +509,12 @@ : (KW_DESCRIBE|KW_DESC) (isExtended=KW_EXTENDED)? (parttype=partTypeExpr) -> ^(TOK_DESCTABLE $parttype $isExtended?) | (KW_DESCRIBE|KW_DESC) KW_FUNCTION KW_EXTENDED? (name=descFuncNames) -> ^(TOK_DESCFUNCTION $name KW_EXTENDED?) ; + +analyzeStatement +@init { msgs.push("analyze statement"); } +@after { msgs.pop(); } + : KW_ANALYZE KW_TABLE (parttype=partTypeExpr) KW_COMPUTE KW_STATISTICS -> ^(TOK_ANALYZE $parttype) + ; showStatement @init { msgs.push("show statement"); } @@ -1728,6 +1736,8 @@ KW_TOUCH: 'TOUCH'; KW_ARCHIVE: 'ARCHIVE'; KW_UNARCHIVE: 'UNARCHIVE'; +KW_COMPUTE: 'COMPUTE'; +KW_STATISTICS: 'STATISTICS'; // Operators // NOTE: if you add a new function/operator, add it to sysFuncNames so that describe function _FUNC_ will work. Index: ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java (revision 982011) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java (working copy) @@ -27,7 +27,7 @@ /** * Implementation of the query block. - * + * **/ public class QB { @@ -170,7 +170,7 @@ } public boolean isSelectStarQuery() { - return qbp.isSelectStarQuery() && aliasToSubq.isEmpty() && !isCTAS(); + return qbp.isSelectStarQuery() && aliasToSubq.isEmpty() && !isCTAS() && !qbp.isAnalyzeCommand(); } public CreateTableDesc getTableDesc() { Index: ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (revision 982011) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (working copy) @@ -558,7 +558,7 @@ public tableSpec(Hive db, HiveConf conf, ASTNode ast) throws SemanticException { - assert (ast.getToken().getType() == HiveParser.TOK_TAB); + assert (ast.getToken().getType() == HiveParser.TOK_TAB || ast.getToken().getType() == HiveParser.TOK_TABTYPE); int childIndex = 0; numDynParts = 0; Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 982011) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy) @@ -23,8 +23,8 @@ import static org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT; import static org.apache.hadoop.util.StringUtils.stringifyException; +import java.io.IOException; import java.io.Serializable; -import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -32,15 +32,15 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.TreeSet; -import java.util.Map.Entry; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.JavaUtils; @@ -49,17 +49,16 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.ql.Context; -import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.ConditionalTask; +import org.apache.hadoop.hive.ql.exec.ExecDriver; import org.apache.hadoop.hive.ql.exec.FetchTask; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.FunctionInfo; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.GroupByOperator; import org.apache.hadoop.hive.ql.exec.JoinOperator; -import org.apache.hadoop.hive.ql.exec.ExecDriver; import org.apache.hadoop.hive.ql.exec.MapRedTask; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.OperatorFactory; @@ -94,6 +93,7 @@ import org.apache.hadoop.hive.ql.optimizer.GenMRFileSink1; import org.apache.hadoop.hive.ql.optimizer.GenMROperator; import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext; +import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx; import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink1; import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink2; import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink3; @@ -103,7 +103,6 @@ import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils; import org.apache.hadoop.hive.ql.optimizer.MapJoinFactory; import org.apache.hadoop.hive.ql.optimizer.Optimizer; -import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx; import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalContext; import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalOptimizer; import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner; @@ -123,6 +122,7 @@ import org.apache.hadoop.hive.ql.plan.FetchWork; import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.FilterDesc; +import org.apache.hadoop.hive.ql.plan.FilterDesc.sampleDesc; import org.apache.hadoop.hive.ql.plan.ForwardDesc; import org.apache.hadoop.hive.ql.plan.GroupByDesc; import org.apache.hadoop.hive.ql.plan.JoinCondDesc; @@ -144,12 +144,11 @@ import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.ql.plan.UDTFDesc; import org.apache.hadoop.hive.ql.plan.UnionDesc; -import org.apache.hadoop.hive.ql.plan.FilterDesc.sampleDesc; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFHash; import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode; import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe; @@ -157,9 +156,9 @@ import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; @@ -266,7 +265,7 @@ @SuppressWarnings("nls") public void doPhase1QBExpr(ASTNode ast, QBExpr qbexpr, String id, String alias) - throws SemanticException { + throws SemanticException { assert (ast.getToken() != null); switch (ast.getToken().getType()) { @@ -276,7 +275,7 @@ qbexpr.setOpcode(QBExpr.Opcode.NULLOP); qbexpr.setQB(qb); } - break; + break; case HiveParser.TOK_UNION: { qbexpr.setOpcode(QBExpr.Opcode.UNION); // query 1 @@ -293,7 +292,7 @@ alias + "-subquery2"); qbexpr.setQBExpr2(qbexpr2); } - break; + break; } } @@ -415,9 +414,9 @@ qb.getParseInfo().setTabSample( alias, new TableSample( - unescapeIdentifier(sampleClause.getChild(0).getText()), - unescapeIdentifier(sampleClause.getChild(1).getText()), - sampleCols)); + unescapeIdentifier(sampleClause.getChild(0).getText()), + unescapeIdentifier(sampleClause.getChild(1).getText()), + sampleCols)); if (unparseTranslator.isEnabled()) { for (ASTNode sampleCol : sampleCols) { unparseTranslator.addIdentifierTranslation((ASTNode) sampleCol @@ -527,7 +526,7 @@ */ private String processLateralView(QB qb, ASTNode lateralView) - throws SemanticException { + throws SemanticException { int numChildren = lateralView.getChildCount(); assert (numChildren == 2); @@ -556,13 +555,13 @@ /** * Phase 1: (including, but not limited to): * - * 1. Gets all the aliases for all the tables / subqueries and makes the - * appropriate mapping in aliasToTabs, aliasToSubq 2. Gets the location of the - * destination and names the clase "inclause" + i 3. Creates a map from a - * string representation of an aggregation tree to the actual aggregation AST - * 4. Creates a mapping from the clause name to the select expression AST in - * destToSelExpr 5. Creates a mapping from a table alias to the lateral view - * AST's in aliasToLateralViews + * 1. Gets all the aliases for all the tables / subqueries and makes the appropriate mapping in + * aliasToTabs, aliasToSubq + * 2. Gets the location of the destination and names the clase "inclause" + i + * 3. Creates a map from a string representation of an aggregation tree to the actual aggregation + * AST + * 4. Creates a mapping from the clause name to the select expression AST in destToSelExpr + * 5. Creates a mapping from a table alias to the lateral view AST's in aliasToLateralViews * * @param ast * @param qb @@ -571,7 +570,7 @@ */ @SuppressWarnings({"fallthrough", "nls"}) public void doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) - throws SemanticException { + throws SemanticException { QBParseInfo qbp = qb.getParseInfo(); boolean skipRecursion = false; @@ -696,6 +695,16 @@ qbp.setDestLimit(ctx_1.dest, new Integer(ast.getChild(0).getText())); break; + case HiveParser.TOK_ANALYZE: + // Case of analyze command + ctx.setGatheringStats(); + String table_name = unescapeIdentifier(ast.getChild(0).getChild(0).getText()); + qb.setTabAlias(table_name, table_name); + qb.getParseInfo().setIsAnalyzeCommand(true); + HiveConf.setVar(conf, HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict"); + HiveConf.setVar(conf, HiveConf.ConfVars.HIVEMAPREDMODE, "nonstrict"); + break; + case HiveParser.TOK_UNION: // currently, we dont support subq1 union subq2 - the user has to // explicitly say: @@ -733,7 +742,6 @@ @SuppressWarnings("nls") public void getMetaData(QB qb) throws SemanticException { try { - LOG.info("Get metadata for source tables"); // Go over the tables and populate the related structures. @@ -759,6 +767,11 @@ .getMsg(qb.getParseInfo().getSrcForAlias(alias))); } + if (qb.getParseInfo().isAnalyzeCommand()) { + tableSpec ts = new tableSpec(db, conf, (ASTNode) ast.getChild(0)); + qb.getParseInfo().addTableSpec(alias, ts); + } + qb.getMetaData().setSrcForAlias(alias, tab); } @@ -803,6 +816,12 @@ // This is a partition qb.getMetaData().setDestForAlias(name, ts.partHandle); } + if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { + // Set that variable to automatically collect stats during the MapReduce job + qb.getParseInfo().setIsInsertToTable(true); + // Add the table spec for the destination table. + qb.getParseInfo().addTableSpec(ts.tableName.toLowerCase(), ts); + } break; } @@ -820,11 +839,11 @@ String location = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE); try { fname = ctx.getExternalTmpFileURI - (FileUtils.makeQualified(new Path(location), conf).toUri()); + (FileUtils.makeQualified(new Path(location), conf).toUri()); } catch (Exception e) { throw new SemanticException("Error creating temporary folder on: " - + location, e); + + location, e); } } else { @@ -857,7 +876,7 @@ ASTNode viewTree; final ASTNodeOrigin viewOrigin = new ASTNodeOrigin("VIEW", tab.getTableName(), tab.getViewExpandedText(), alias, qb.getParseInfo().getSrcForAlias( - alias)); + alias)); try { String viewText = tab.getViewExpandedText(); // Reparse text, passing null for context to avoid clobbering @@ -935,7 +954,7 @@ // whether it is or not if (fields != null) { fields - .add(unescapeIdentifier(condn.getToken().getText().toLowerCase())); + .add(unescapeIdentifier(condn.getToken().getText().toLowerCase())); } unparseTranslator.addIdentifierTranslation(condn); break; @@ -1122,7 +1141,7 @@ for (int ci = childrenBegin; ci < joinCond.getChildCount(); ci++) { parseJoinCondPopulateAlias(joinTree, (ASTNode) joinCond.getChild(ci), leftAlias.get(ci - childrenBegin), rightAlias.get(ci - - childrenBegin), null); + - childrenBegin), null); } boolean leftAliasNull = true; @@ -1167,7 +1186,7 @@ @SuppressWarnings("nls") private Operator genFilterPlan(String dest, QB qb, Operator input) - throws SemanticException { + throws SemanticException { ASTNode whereExpr = qb.getParseInfo().getWhrForClause(dest); return genFilterPlan(qb, (ASTNode) whereExpr.getChild(0), input); @@ -1185,17 +1204,18 @@ */ @SuppressWarnings("nls") private Operator genFilterPlan(QB qb, ASTNode condn, Operator input) - throws SemanticException { + throws SemanticException { OpParseContext inputCtx = opParseCtx.get(input); RowResolver inputRR = inputCtx.getRR(); Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild( new FilterDesc(genExprNodeDesc(condn, inputRR), false), new RowSchema( - inputRR.getColumnInfos()), input), inputRR); + inputRR.getColumnInfos()), input), inputRR); - if (LOG.isDebugEnabled()) + if (LOG.isDebugEnabled()) { LOG.debug("Created Filter Plan for " + qb.getId() + " row schema: " - + inputRR.toString()); + + inputRR.toString()); + } return output; } @@ -1203,7 +1223,7 @@ private Integer genColListRegex(String colRegex, String tabAlias, String alias, ASTNode sel, ArrayList col_list, RowResolver input, Integer pos, RowResolver output) - throws SemanticException { + throws SemanticException { // The table alias should exist if (tabAlias != null && !input.hasTableAlias(tabAlias)) { @@ -1232,7 +1252,7 @@ if (tabAlias != null && !tmp[0].equalsIgnoreCase(tabAlias)) { continue; } - + if(colInfo.getIsVirtualCol() && colInfo.isHiddenVirtualCol()) { continue; } @@ -1247,7 +1267,7 @@ col_list.add(expr); output.put(tmp[0], tmp[1], new ColumnInfo(getColumnInternalName(pos), colInfo.getType(), colInfo - .getTabAlias(), colInfo.getIsVirtualCol(), colInfo.isHiddenVirtualCol())); + .getTabAlias(), colInfo.getIsVirtualCol(), colInfo.isHiddenVirtualCol())); pos = Integer.valueOf(pos.intValue() + 1); matched++; @@ -1287,7 +1307,7 @@ // the reference to the original file path if (ss.getConf().get("mapred.job.tracker", "local").equals("local")) { Set files = ss - .list_resource(SessionState.ResourceType.FILE, null); + .list_resource(SessionState.ResourceType.FILE, null); if ((files != null) && !files.isEmpty()) { int end = cmd.indexOf(" "); String prog = (end == -1) ? cmd : cmd.substring(0, end); @@ -1343,7 +1363,7 @@ case HiveParser.TOK_TABLEROWFORMATFIELD: String fieldDelim = unescapeSQLString(rowChild.getChild(0).getText()); tblDesc.getProperties() - .setProperty(Constants.FIELD_DELIM, fieldDelim); + .setProperty(Constants.FIELD_DELIM, fieldDelim); tblDesc.getProperties().setProperty(Constants.SERIALIZATION_FORMAT, fieldDelim); @@ -1383,7 +1403,7 @@ } private void failIfColAliasExists(Set nameSet, String name) - throws SemanticException { + throws SemanticException { if (nameSet.contains(name)) { throw new SemanticException(ErrorMsg.COLUMN_ALIAS_ALREADY_EXISTS .getMsg(name)); @@ -1393,7 +1413,7 @@ @SuppressWarnings("nls") private Operator genScriptPlan(ASTNode trfm, QB qb, Operator input) - throws SemanticException { + throws SemanticException { // If there is no "AS" clause, the output schema will be "key,value" ArrayList outputCols = new ArrayList(); int inputSerDeNum = 1, inputRecordWriterNum = 2; @@ -1452,7 +1472,7 @@ String intName = getColumnInternalName(i); ColumnInfo colInfo = new ColumnInfo(intName, TypeInfoUtils .getTypeInfoFromTypeString(getTypeStringFromAST((ASTNode) child - .getChild(1))), null, false); + .getChild(1))), null, false); colInfo.setAlias(colAlias); outputCols.add(colInfo); } @@ -1479,7 +1499,7 @@ StringBuilder inpColumns = new StringBuilder(); StringBuilder inpColumnTypes = new StringBuilder(); ArrayList inputSchema = opParseCtx.get(input).getRR() - .getColumnInfos(); + .getColumnInfos(); for (int i = 0; i < inputSchema.size(); ++i) { if (i != 0) { inpColumns.append(","); @@ -1540,15 +1560,15 @@ Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild( new ScriptDesc( - getFixedCmd(stripQuotes(trfm.getChild(execPos).getText())), inInfo, - inRecordWriter, outInfo, outRecordReader, errRecordReader, errInfo), - new RowSchema(out_rwsch.getColumnInfos()), input), out_rwsch); + getFixedCmd(stripQuotes(trfm.getChild(execPos).getText())), inInfo, + inRecordWriter, outInfo, outRecordReader, errRecordReader, errInfo), + new RowSchema(out_rwsch.getColumnInfos()), input), out_rwsch); return output; } private Class getRecordReader(ASTNode node) - throws SemanticException { + throws SemanticException { String name; if (node.getChildCount() == 0) { @@ -1566,7 +1586,7 @@ } private Class getDefaultRecordReader() - throws SemanticException { + throws SemanticException { String name; name = conf.getVar(HiveConf.ConfVars.HIVESCRIPTRECORDREADER); @@ -1580,7 +1600,7 @@ } private Class getRecordWriter(ASTNode node) - throws SemanticException { + throws SemanticException { String name; if (node.getChildCount() == 0) { @@ -1692,13 +1712,14 @@ } private Operator genSelectPlan(String dest, QB qb, Operator input) - throws SemanticException { + throws SemanticException { ASTNode selExprList = qb.getParseInfo().getSelForClause(dest); Operator op = genSelectPlan(selExprList, qb, input); - if (LOG.isDebugEnabled()) + if (LOG.isDebugEnabled()) { LOG.debug("Created Select Plan for clause: " + dest); + } return op; } @@ -1707,8 +1728,9 @@ private Operator genSelectPlan(ASTNode selExprList, QB qb, Operator input) throws SemanticException { - if (LOG.isDebugEnabled()) + if (LOG.isDebugEnabled()) { LOG.debug("tree: " + selExprList.toStringTree()); + } ArrayList col_list = new ArrayList(); RowResolver out_rwsch = new RowResolver(); @@ -1744,7 +1766,7 @@ if (udtfExprType == HiveParser.TOK_FUNCTION || udtfExprType == HiveParser.TOK_FUNCTIONSTAR) { String funcName = TypeCheckProcFactory.DefaultExprProcessor - .getFunctionText(udtfExpr, true); + .getFunctionText(udtfExpr, true); FunctionInfo fi = FunctionRegistry.getFunctionInfo(funcName); if (fi != null) { genericUDTF = fi.getGenericUDTF(); @@ -1802,8 +1824,9 @@ exprList = selExprList; } - if (LOG.isDebugEnabled()) + if (LOG.isDebugEnabled()) { LOG.debug("genSelectPlan: input = " + inputRR.toString()); + } // For UDTF's, skip the function name to get the expressions int startPosn = isUDTF ? posn + 1 : posn; @@ -1862,15 +1885,15 @@ } else if (expr.getType() == HiveParser.DOT && expr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL && inputRR.hasTableAlias(unescapeIdentifier(expr.getChild(0) - .getChild(0).getText().toLowerCase())) && !hasAsClause - && !inputRR.getIsExprResolver() - && isRegex(unescapeIdentifier(expr.getChild(1).getText()))) { + .getChild(0).getText().toLowerCase())) && !hasAsClause + && !inputRR.getIsExprResolver() + && isRegex(unescapeIdentifier(expr.getChild(1).getText()))) { // In case the expression is TABLE.COL (col can be regex). // This can only happen without AS clause // We don't allow this for ExprResolver - the Group By case pos = genColListRegex(unescapeIdentifier(expr.getChild(1).getText()), unescapeIdentifier(expr.getChild(0).getChild(0).getText() - .toLowerCase()), alias, expr, col_list, inputRR, pos, out_rwsch); + .toLowerCase()), alias, expr, col_list, inputRR, pos, out_rwsch); } else { // Case when this is an expression ExprNodeDesc exp = genExprNodeDesc(expr, inputRR); @@ -1902,7 +1925,7 @@ Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild( new SelectDesc(col_list, columnNames, selectStar), new RowSchema( - out_rwsch.getColumnInfos()), input), out_rwsch); + out_rwsch.getColumnInfos()), input), out_rwsch); output.setColumnExprMap(colExprMap); if (isInTransform) { @@ -1913,8 +1936,9 @@ output = genUDTFPlan(genericUDTF, udtfTableAlias, udtfColAliases, qb, output); } - if (LOG.isDebugEnabled()) + if (LOG.isDebugEnabled()) { LOG.debug("Created Select Plan row schema: " + out_rwsch.toString()); + } return output; } @@ -1945,7 +1969,7 @@ ObjectInspector[] result = new ObjectInspector[exprs.size()]; for (int i = 0; i < exprs.size(); i++) { result[i] = TypeInfoUtils - .getStandardWritableObjectInspectorFromTypeInfo(exprs.get(i)); + .getStandardWritableObjectInspectorFromTypeInfo(exprs.get(i)); } return result; } @@ -1957,13 +1981,13 @@ static GenericUDAFEvaluator getGenericUDAFEvaluator(String aggName, ArrayList aggParameters, ASTNode aggTree, boolean isDistinct, boolean isAllColumns) - throws SemanticException { + throws SemanticException { ArrayList originalParameterTypeInfos = getTypeInfo(aggParameters); GenericUDAFEvaluator result = FunctionRegistry.getGenericUDAFEvaluator( aggName, originalParameterTypeInfos, isDistinct, isAllColumns); if (null == result) { String reason = "Looking for UDAF Evaluator\"" + aggName - + "\" with parameters " + originalParameterTypeInfos; + + "\" with parameters " + originalParameterTypeInfos; throw new SemanticException(ErrorMsg.INVALID_FUNCTION_SIGNATURE.getMsg( (ASTNode) aggTree.getChild(0), reason)); } @@ -1985,7 +2009,7 @@ */ static GenericUDAFInfo getGenericUDAFInfo(GenericUDAFEvaluator evaluator, GenericUDAFEvaluator.Mode emode, ArrayList aggParameters) - throws SemanticException { + throws SemanticException { GenericUDAFInfo r = new GenericUDAFInfo(); @@ -1996,7 +2020,7 @@ ObjectInspector returnOI = null; try { ObjectInspector[] aggObjectInspectors = - getStandardObjectInspector(getTypeInfo(aggParameters)); + getStandardObjectInspector(getTypeInfo(aggParameters)); returnOI = r.genericUDAFEvaluator.init(emode, aggObjectInspectors); r.returnType = TypeInfoUtils.getTypeInfoFromObjectInspector(returnOI); } catch (HiveException e) { @@ -2049,9 +2073,9 @@ private Operator genGroupByPlanGroupByOperator(QBParseInfo parseInfo, String dest, Operator reduceSinkOperatorInfo, GroupByDesc.Mode mode, Map genericUDAFEvaluators) - throws SemanticException { + throws SemanticException { RowResolver groupByInputRowResolver = opParseCtx - .get(reduceSinkOperatorInfo).getRR(); + .get(reduceSinkOperatorInfo).getRR(); RowResolver groupByOutputRowResolver = new RowResolver(); groupByOutputRowResolver.setIsExprResolver(true); ArrayList groupByKeys = new ArrayList(); @@ -2077,7 +2101,7 @@ } // For each aggregation HashMap aggregationTrees = parseInfo - .getAggregationExprsForClause(dest); + .getAggregationExprsForClause(dest); assert (aggregationTrees != null); for (Map.Entry entry : aggregationTrees.entrySet()) { ASTNode value = entry.getValue(); @@ -2128,8 +2152,8 @@ Operator op = putOpInsertMap(OperatorFactory.getAndMakeChild( new GroupByDesc(mode, outputColumnNames, groupByKeys, aggregations, - false), new RowSchema(groupByOutputRowResolver.getColumnInfos()), - reduceSinkOperatorInfo), groupByOutputRowResolver); + false), new RowSchema(groupByOutputRowResolver.getColumnInfos()), + reduceSinkOperatorInfo), groupByOutputRowResolver); op.setColumnExprMap(colExprMap); return op; } @@ -2154,7 +2178,7 @@ boolean distPartAgg) throws SemanticException { ArrayList outputColumnNames = new ArrayList(); RowResolver groupByInputRowResolver = opParseCtx - .get(reduceSinkOperatorInfo).getRR(); + .get(reduceSinkOperatorInfo).getRR(); RowResolver groupByOutputRowResolver = new RowResolver(); groupByOutputRowResolver.setIsExprResolver(true); ArrayList groupByKeys = new ArrayList(); @@ -2180,7 +2204,7 @@ } HashMap aggregationTrees = parseInfo - .getAggregationExprsForClause(dest); + .getAggregationExprsForClause(dest); for (Map.Entry entry : aggregationTrees.entrySet()) { ASTNode value = entry.getValue(); String aggName = value.getChild(0).getText(); @@ -2255,9 +2279,9 @@ Operator op = putOpInsertMap(OperatorFactory.getAndMakeChild( new GroupByDesc(mode, outputColumnNames, groupByKeys, aggregations, - distPartAgg), new RowSchema(groupByOutputRowResolver - .getColumnInfos()), reduceSinkOperatorInfo), - groupByOutputRowResolver); + distPartAgg), new RowSchema(groupByOutputRowResolver + .getColumnInfos()), reduceSinkOperatorInfo), + groupByOutputRowResolver); op.setColumnExprMap(colExprMap); return op; } @@ -2279,10 +2303,10 @@ private Operator genGroupByPlanMapGroupByOperator(QB qb, String dest, Operator inputOperatorInfo, GroupByDesc.Mode mode, Map genericUDAFEvaluators) - throws SemanticException { + throws SemanticException { RowResolver groupByInputRowResolver = opParseCtx.get(inputOperatorInfo) - .getRR(); + .getRR(); QBParseInfo parseInfo = qb.getParseInfo(); RowResolver groupByOutputRowResolver = new RowResolver(); groupByOutputRowResolver.setIsExprResolver(true); @@ -2327,7 +2351,7 @@ // For each aggregation HashMap aggregationTrees = parseInfo - .getAggregationExprsForClause(dest); + .getAggregationExprsForClause(dest); assert (aggregationTrees != null); for (Map.Entry entry : aggregationTrees.entrySet()) { @@ -2370,8 +2394,8 @@ Operator op = putOpInsertMap(OperatorFactory.getAndMakeChild( new GroupByDesc(mode, outputColumnNames, groupByKeys, aggregations, - false), new RowSchema(groupByOutputRowResolver.getColumnInfos()), - inputOperatorInfo), groupByOutputRowResolver); + false), new RowSchema(groupByOutputRowResolver.getColumnInfos()), + inputOperatorInfo), groupByOutputRowResolver); op.setColumnExprMap(colExprMap); return op; } @@ -2396,7 +2420,7 @@ boolean mapAggrDone) throws SemanticException { RowResolver reduceSinkInputRowResolver = opParseCtx.get(inputOperatorInfo) - .getRR(); + .getRR(); QBParseInfo parseInfo = qb.getParseInfo(); RowResolver reduceSinkOutputRowResolver = new RowResolver(); reduceSinkOutputRowResolver.setIsExprResolver(true); @@ -2414,7 +2438,7 @@ if (reduceSinkOutputRowResolver.getExpression(grpbyExpr) == null) { outputColumnNames.add(getColumnInternalName(reduceKeys.size() - 1)); String field = Utilities.ReduceField.KEY.toString() + "." - + getColumnInternalName(reduceKeys.size() - 1); + + getColumnInternalName(reduceKeys.size() - 1); ColumnInfo colInfo = new ColumnInfo(field, reduceKeys.get( reduceKeys.size() - 1).getTypeInfo(), null, false); reduceSinkOutputRowResolver.putExpression(grpbyExpr, colInfo); @@ -2433,10 +2457,10 @@ ASTNode parameter = (ASTNode) value.getChild(i); if (reduceSinkOutputRowResolver.getExpression(parameter) == null) { reduceKeys - .add(genExprNodeDesc(parameter, reduceSinkInputRowResolver)); + .add(genExprNodeDesc(parameter, reduceSinkInputRowResolver)); outputColumnNames.add(getColumnInternalName(reduceKeys.size() - 1)); String field = Utilities.ReduceField.KEY.toString() + "." - + getColumnInternalName(reduceKeys.size() - 1); + + getColumnInternalName(reduceKeys.size() - 1); ColumnInfo colInfo = new ColumnInfo(field, reduceKeys.get( reduceKeys.size() - 1).getTypeInfo(), null, false); reduceSinkOutputRowResolver.putExpression(parameter, colInfo); @@ -2448,7 +2472,7 @@ ArrayList reduceValues = new ArrayList(); HashMap aggregationTrees = parseInfo - .getAggregationExprsForClause(dest); + .getAggregationExprsForClause(dest); if (!mapAggrDone) { // Put parameters to aggregations in reduceValues @@ -2461,9 +2485,9 @@ reduceValues.add(genExprNodeDesc(parameter, reduceSinkInputRowResolver)); outputColumnNames - .add(getColumnInternalName(reduceValues.size() - 1)); + .add(getColumnInternalName(reduceValues.size() - 1)); String field = Utilities.ReduceField.VALUE.toString() + "." - + getColumnInternalName(reduceValues.size() - 1); + + getColumnInternalName(reduceValues.size() - 1); reduceSinkOutputRowResolver.putExpression(parameter, new ColumnInfo(field, reduceValues.get(reduceValues.size() - 1).getTypeInfo(), null, false)); @@ -2483,7 +2507,7 @@ inputField++; outputColumnNames.add(getColumnInternalName(reduceValues.size() - 1)); String field = Utilities.ReduceField.VALUE.toString() + "." - + getColumnInternalName(reduceValues.size() - 1); + + getColumnInternalName(reduceValues.size() - 1); reduceSinkOutputRowResolver.putExpression(entry.getValue(), new ColumnInfo(field, type, null, false)); } @@ -2491,9 +2515,9 @@ ReduceSinkOperator rsOp = (ReduceSinkOperator) putOpInsertMap( OperatorFactory.getAndMakeChild(PlanUtils.getReduceSinkDesc(reduceKeys, - reduceValues, outputColumnNames, true, -1, numPartitionFields, - numReducers), new RowSchema(reduceSinkOutputRowResolver - .getColumnInfos()), inputOperatorInfo), reduceSinkOutputRowResolver); + reduceValues, outputColumnNames, true, -1, numPartitionFields, + numReducers), new RowSchema(reduceSinkOutputRowResolver + .getColumnInfos()), inputOperatorInfo), reduceSinkOutputRowResolver); rsOp.setColumnExprMap(colExprMap); return rsOp; } @@ -2545,12 +2569,12 @@ ArrayList reduceValues = new ArrayList(); int inputField = reduceKeys.size(); HashMap aggregationTrees = parseInfo - .getAggregationExprsForClause(dest); + .getAggregationExprsForClause(dest); for (Map.Entry entry : aggregationTrees.entrySet()) { String field = getColumnInternalName(inputField); ASTNode t = entry.getValue(); TypeInfo typeInfo = reduceSinkInputRowResolver2.getExpression(t) - .getType(); + .getType(); reduceValues.add(new ExprNodeColumnDesc(typeInfo, field, "", false)); inputField++; String col = getColumnInternalName(reduceValues.size() - 1); @@ -2562,10 +2586,10 @@ ReduceSinkOperator rsOp = (ReduceSinkOperator) putOpInsertMap( OperatorFactory.getAndMakeChild(PlanUtils.getReduceSinkDesc(reduceKeys, - reduceValues, outputColumnNames, true, -1, numPartitionFields, - numReducers), new RowSchema(reduceSinkOutputRowResolver2 - .getColumnInfos()), groupByOperatorInfo), - reduceSinkOutputRowResolver2); + reduceValues, outputColumnNames, true, -1, numPartitionFields, + numReducers), new RowSchema(reduceSinkOutputRowResolver2 + .getColumnInfos()), groupByOperatorInfo), + reduceSinkOutputRowResolver2); rsOp.setColumnExprMap(colExprMap); return rsOp; @@ -2588,7 +2612,7 @@ private Operator genGroupByPlanGroupByOperator2MR(QBParseInfo parseInfo, String dest, Operator reduceSinkOperatorInfo2, GroupByDesc.Mode mode, Map genericUDAFEvaluators) - throws SemanticException { + throws SemanticException { RowResolver groupByInputRowResolver2 = opParseCtx.get( reduceSinkOperatorInfo2).getRR(); RowResolver groupByOutputRowResolver2 = new RowResolver(); @@ -2615,7 +2639,7 @@ colExprMap.put(field, groupByKeys.get(groupByKeys.size() - 1)); } HashMap aggregationTrees = parseInfo - .getAggregationExprsForClause(dest); + .getAggregationExprsForClause(dest); for (Map.Entry entry : aggregationTrees.entrySet()) { ArrayList aggParameters = new ArrayList(); ASTNode value = entry.getValue(); @@ -2635,18 +2659,18 @@ boolean isStar = value.getType() == HiveParser.TOK_FUNCTIONSTAR; Mode amode = groupByDescModeToUDAFMode(mode, isDistinct); GenericUDAFEvaluator genericUDAFEvaluator = genericUDAFEvaluators - .get(entry.getKey()); + .get(entry.getKey()); assert (genericUDAFEvaluator != null); GenericUDAFInfo udaf = getGenericUDAFInfo(genericUDAFEvaluator, amode, aggParameters); aggregations - .add(new AggregationDesc( - aggName.toLowerCase(), - udaf.genericUDAFEvaluator, - udaf.convertedParameters, - (mode != GroupByDesc.Mode.FINAL && value.getToken().getType() == - HiveParser.TOK_FUNCTIONDI), - amode)); + .add(new AggregationDesc( + aggName.toLowerCase(), + udaf.genericUDAFEvaluator, + udaf.convertedParameters, + (mode != GroupByDesc.Mode.FINAL && value.getToken().getType() == + HiveParser.TOK_FUNCTIONDI), + amode)); String field = getColumnInternalName(groupByKeys.size() + aggregations.size() - 1); outputColumnNames.add(field); @@ -2656,8 +2680,8 @@ Operator op = putOpInsertMap(OperatorFactory.getAndMakeChild( new GroupByDesc(mode, outputColumnNames, groupByKeys, aggregations, - false), new RowSchema(groupByOutputRowResolver2.getColumnInfos()), - reduceSinkOperatorInfo2), groupByOutputRowResolver2); + false), new RowSchema(groupByOutputRowResolver2.getColumnInfos()), + reduceSinkOperatorInfo2), groupByOutputRowResolver2); op.setColumnExprMap(colExprMap); return op; } @@ -2689,7 +2713,7 @@ **/ @SuppressWarnings({"unused", "nls"}) private Operator genGroupByPlan1MR(String dest, QB qb, Operator input) - throws SemanticException { + throws SemanticException { QBParseInfo parseInfo = qb.getParseInfo(); @@ -2819,7 +2843,7 @@ */ @SuppressWarnings("nls") private Operator genGroupByPlan2MR(String dest, QB qb, Operator input) - throws SemanticException { + throws SemanticException { QBParseInfo parseInfo = qb.getParseInfo(); @@ -2832,7 +2856,7 @@ // captured by WritableComparableHiveObject.hashCode() function. Operator reduceSinkOperatorInfo = genGroupByPlanReduceSinkOperator(qb, dest, input, (parseInfo.getDistinctFuncExprForClause(dest) == null ? -1 - : Integer.MAX_VALUE), -1, false); + : Integer.MAX_VALUE), -1, false); // ////// 2. Generate GroupbyOperator Map genericUDAFEvaluators = @@ -2984,8 +3008,8 @@ // ////// Generate ReduceSink Operator Operator reduceSinkOperatorInfo = genGroupByPlanReduceSinkOperator(qb, dest, groupByOperatorInfo, (parseInfo - .getDistinctFuncExprForClause(dest) == null ? -1 - : Integer.MAX_VALUE), -1, true); + .getDistinctFuncExprForClause(dest) == null ? -1 + : Integer.MAX_VALUE), -1, true); // ////// Generate GroupbyOperator for a partial aggregation Operator groupByOperatorInfo2 = genGroupByPlanGroupByOperator1(parseInfo, @@ -3019,7 +3043,7 @@ @SuppressWarnings("nls") private Operator genConversionOps(String dest, QB qb, Operator input) - throws SemanticException { + throws SemanticException { Integer dest_type = qb.getMetaData().getDestTypeForAlias(dest); switch (dest_type.intValue()) { @@ -3040,10 +3064,10 @@ } private int getReducersBucketing(int totalFiles, int maxReducers) { - int numFiles = totalFiles/maxReducers; + int numFiles = totalFiles / maxReducers; while (true) { - if (totalFiles%numFiles == 0) { - return totalFiles/numFiles; + if (totalFiles % numFiles == 0) { + return totalFiles / numFiles; } numFiles++; } @@ -3052,8 +3076,8 @@ private static class SortBucketRSCtx { ArrayList partnCols; boolean multiFileSpray; - int numFiles; - int totalFiles; + int numFiles; + int totalFiles; public SortBucketRSCtx() { partnCols = null; @@ -3070,7 +3094,8 @@ } /** - * @param partnCols the partnCols to set + * @param partnCols + * the partnCols to set */ public void setPartnCols(ArrayList partnCols) { this.partnCols = partnCols; @@ -3084,7 +3109,8 @@ } /** - * @param multiFileSpray the multiFileSpray to set + * @param multiFileSpray + * the multiFileSpray to set */ public void setMultiFileSpray(boolean multiFileSpray) { this.multiFileSpray = multiFileSpray; @@ -3098,7 +3124,8 @@ } /** - * @param numFiles the numFiles to set + * @param numFiles + * the numFiles to set */ public void setNumFiles(int numFiles) { this.numFiles = numFiles; @@ -3112,7 +3139,8 @@ } /** - * @param totalFiles the totalFiles to set + * @param totalFiles + * the totalFiles to set */ public void setTotalFiles(int totalFiles) { this.totalFiles = totalFiles; @@ -3120,9 +3148,10 @@ } @SuppressWarnings("nls") - private Operator genBucketingSortingDest(String dest, Operator input, QB qb, TableDesc table_desc, - Table dest_tab, SortBucketRSCtx ctx) - throws SemanticException { + private Operator genBucketingSortingDest(String dest, Operator input, QB qb, + TableDesc table_desc, + Table dest_tab, SortBucketRSCtx ctx) + throws SemanticException { // If the table is bucketed, and bucketing is enforced, do the following: // If the number of buckets is smaller than the number of maximum reducers, @@ -3131,19 +3160,20 @@ // spray the data into multiple buckets. That way, we can support a very large // number of buckets without needing a very large number of reducers. boolean enforceBucketing = false; - boolean enforceSorting = false; + boolean enforceSorting = false; ArrayList partnCols = new ArrayList(); ArrayList partnColsNoConvert = new ArrayList(); - ArrayList sortCols = new ArrayList(); + ArrayList sortCols = new ArrayList(); boolean multiFileSpray = false; - int numFiles = 1; - int totalFiles = 1; + int numFiles = 1; + int totalFiles = 1; if ((dest_tab.getNumBuckets() > 0) && (conf.getBoolVar(HiveConf.ConfVars.HIVEENFORCEBUCKETING))) { enforceBucketing = true; partnCols = getParitionColsFromBucketCols(dest, qb, dest_tab, table_desc, input, true); - partnColsNoConvert = getParitionColsFromBucketCols(dest, qb, dest_tab, table_desc, input, false); + partnColsNoConvert = getParitionColsFromBucketCols(dest, qb, dest_tab, table_desc, input, + false); } if ((dest_tab.getSortCols() != null) && @@ -3159,29 +3189,28 @@ if (enforceBucketing || enforceSorting) { int maxReducers = conf.getIntVar(HiveConf.ConfVars.MAXREDUCERS); - int numBuckets = dest_tab.getNumBuckets(); + int numBuckets = dest_tab.getNumBuckets(); if (numBuckets > maxReducers) { multiFileSpray = true; totalFiles = numBuckets; if (totalFiles % maxReducers == 0) { numFiles = totalFiles / maxReducers; - } - else { + } else { // find the number of reducers such that it is a divisor of totalFiles maxReducers = getReducersBucketing(totalFiles, maxReducers); - numFiles = totalFiles/maxReducers; + numFiles = totalFiles / maxReducers; } - } - else { + } else { maxReducers = numBuckets; } - input = genReduceSinkPlanForSortingBucketing(dest_tab, input, sortCols, partnCols, maxReducers); + input = genReduceSinkPlanForSortingBucketing(dest_tab, input, sortCols, partnCols, + maxReducers); ctx.setMultiFileSpray(multiFileSpray); ctx.setNumFiles(numFiles); ctx.setPartnCols(partnColsNoConvert); ctx.setTotalFiles(totalFiles); - //disable "merge mapfiles" and "merge mapred files". + // disable "merge mapfiles" and "merge mapred files". HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVEMERGEMAPFILES, false); HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVEMERGEMAPREDFILES, false); } @@ -3190,6 +3219,7 @@ /** * Check for HOLD_DDLTIME hint. + * * @param qb * @return true if HOLD_DDLTIME is set, false otherwise. */ @@ -3209,13 +3239,13 @@ @SuppressWarnings("nls") private Operator genFileSinkPlan(String dest, QB qb, Operator input) - throws SemanticException { + throws SemanticException { RowResolver inputRR = opParseCtx.get(input).getRR(); QBMetaData qbm = qb.getMetaData(); Integer dest_type = qbm.getDestTypeForAlias(dest); - Table dest_tab = null; // destination table if any + Table dest_tab = null; // destination table if any String queryTmpdir = null; // the intermediate destination directory Path dest_path = null; // the final destination directory TableDesc table_desc = null; @@ -3228,7 +3258,6 @@ switch (dest_type.intValue()) { case QBMetaData.DEST_TABLE: { - dest_tab = qbm.getDestTableForAlias(dest); Map partSpec = qbm.getPartSpecForAlias(dest); dest_path = dest_tab.getPath(); @@ -3236,7 +3265,7 @@ // check for partition List parts = dest_tab.getPartitionKeys(); if (parts != null && parts.size() > 0) { // table is partitioned - if (partSpec== null || partSpec.size() == 0) { // user did NOT specify partition + if (partSpec == null || partSpec.size() == 0) { // user did NOT specify partition throw new SemanticException(ErrorMsg.NEED_PARTITION_ERROR.getMsg()); } // the HOLD_DDLTIIME hint should not be used with dynamic partition since the @@ -3257,7 +3286,7 @@ // TODO: we should support merge files for dynamically generated partitions later if (dpCtx.getNumDPCols() > 0 && (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEMERGEMAPFILES) || - HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEMERGEMAPREDFILES))) { + HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEMERGEMAPREDFILES))) { HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVEMERGEMAPFILES, false); HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVEMERGEMAPREDFILES, false); } @@ -3319,7 +3348,6 @@ break; } case QBMetaData.DEST_PARTITION: { - Partition dest_part = qbm.getDestPartitionForAlias(dest); dest_tab = dest_part.getTable(); @@ -3363,6 +3391,7 @@ isLocal = true; // fall through case QBMetaData.DEST_DFS_FILE: { + dest_path = new Path(qbm.getDestFileForAlias(dest)); String destStr = dest_path.toString(); @@ -3480,13 +3509,13 @@ try { StructObjectInspector rowObjectInspector = (StructObjectInspector) table_desc - .getDeserializer().getObjectInspector(); + .getDeserializer().getObjectInspector(); List fields = rowObjectInspector - .getAllStructFieldRefs(); + .getAllStructFieldRefs(); for (int i = 0; i < fields.size(); i++) { vecCol.add(new ColumnInfo(fields.get(i).getFieldName(), TypeInfoUtils .getTypeInfoFromObjectInspector(fields.get(i) - .getFieldObjectInspector()), "", false)); + .getFieldObjectInspector()), "", false)); } } catch (Exception e) { throw new SemanticException(e.getMessage()); @@ -3494,41 +3523,62 @@ RowSchema fsRS = new RowSchema(vecCol); + FileSinkDesc fileSinkDesc = new FileSinkDesc( + queryTmpdir, + table_desc, + conf.getBoolVar(HiveConf.ConfVars.COMPRESSRESULT), + currentTableId, + rsCtx.isMultiFileSpray(), + rsCtx.getNumFiles(), + rsCtx.getTotalFiles(), + rsCtx.getPartnCols(), + dpCtx); + + if (qb.getParseInfo().isInsertToTable()) { + // The static path of the partitions has to be calculated and passed to the fileSinkDesc + // (used in stats publishing as part of the partition ID) + // I had to check for the tablename for being null as it is null for inner queries + if (table_desc.getTableName() != null) { + String staticSpec = table_desc.getTableName(); + Map partSpec = qb.getParseInfo().getTableSpec(table_desc.getTableName().toLowerCase()).getPartSpec(); + if (partSpec != null) { + for (String key : partSpec.keySet()) { + String val = partSpec.get(key); + if (val != null) { + staticSpec += Path.SEPARATOR + key + "=" + val; + } else { + break; + } + } + } + fileSinkDesc.setStaticSpec(staticSpec); + } + } + Operator output = putOpInsertMap( - OperatorFactory.getAndMakeChild( - new FileSinkDesc( - queryTmpdir, - table_desc, - conf.getBoolVar(HiveConf.ConfVars.COMPRESSRESULT), - currentTableId, - rsCtx.isMultiFileSpray(), - rsCtx.getNumFiles(), - rsCtx.getTotalFiles(), - rsCtx.getPartnCols(), - dpCtx), - fsRS, input), inputRR); + OperatorFactory.getAndMakeChild(fileSinkDesc, fsRS, input), inputRR); - if (ltd != null && SessionState.get() != null) { SessionState.get().getLineageState() - .mapDirToFop(ltd.getSourceDir(), (FileSinkOperator)output); + .mapDirToFop(ltd.getSourceDir(), (FileSinkOperator) output); } - if (LOG.isDebugEnabled()) + if (LOG.isDebugEnabled()) { LOG.debug("Created FileSink Plan for clause: " + dest + "dest_path: " - + dest_path + " row schema: " + inputRR.toString()); + + dest_path + " row schema: " + inputRR.toString()); + } return output; } private void validatePartSpec(Table tbl, Map partSpec) - throws SemanticException { + throws SemanticException { List parts = tbl.getPartitionKeys(); Set partCols = new HashSet(parts.size()); - for (FieldSchema col: parts) { + for (FieldSchema col : parts) { partCols.add(col.getName()); } - for (String col: partSpec.keySet()) { + for (String col : partSpec.keySet()) { if (!partCols.contains(col)) { throw new SemanticException(ErrorMsg.NONEXISTPARTCOL.getMsg()); } @@ -3544,7 +3594,7 @@ StructObjectInspector oi = null; try { Deserializer deserializer = table_desc.getDeserializerClass() - .newInstance(); + .newInstance(); deserializer.initialize(conf, table_desc.getProperties()); oi = (StructObjectInspector) deserializer.getObjectInspector(); } catch (Exception e) { @@ -3555,19 +3605,19 @@ List tableFields = oi.getAllStructFieldRefs(); boolean dynPart = HiveConf.getBoolVar(conf, HiveConf.ConfVars.DYNAMICPARTITIONING); ArrayList rowFields = opParseCtx.get(input).getRR() - .getColumnInfos(); + .getColumnInfos(); int inColumnCnt = rowFields.size(); int outColumnCnt = tableFields.size(); if (dynPart && dpCtx != null) { - outColumnCnt += dpCtx.getNumDPCols(); + outColumnCnt += dpCtx.getNumDPCols(); } if (inColumnCnt != outColumnCnt) { String reason = "Table " + dest + " has " + outColumnCnt - + " columns, but query has " + inColumnCnt + " columns."; + + " columns, but query has " + inColumnCnt + " columns."; throw new SemanticException(ErrorMsg.TARGET_TABLE_COLUMN_MISMATCH.getMsg( qb.getParseInfo().getDestForClause(dest), reason)); - } else if (dynPart && dpCtx != null){ + } else if (dynPart && dpCtx != null) { // create the mapping from input ExprNode to dest table DP column dpCtx.mapInputToDP(rowFields.subList(tableFields.size(), rowFields.size())); } @@ -3588,9 +3638,9 @@ // here only deals with non-partition columns. We deal with partition columns next for (int i = 0; i < columnNumber; i++) { ObjectInspector tableFieldOI = tableFields.get(i) - .getFieldObjectInspector(); + .getFieldObjectInspector(); TypeInfo tableFieldTypeInfo = TypeInfoUtils - .getTypeInfoFromObjectInspector(tableFieldOI); + .getTypeInfoFromObjectInspector(tableFieldOI); TypeInfo rowFieldTypeInfo = rowFields.get(i).getType(); ExprNodeDesc column = new ExprNodeColumnDesc(rowFieldTypeInfo, rowFields.get(i).getInternalName(), "", false); @@ -3598,8 +3648,8 @@ // JSON-format. if (!tableFieldTypeInfo.equals(rowFieldTypeInfo) && !(isLazySimpleSerDe - && tableFieldTypeInfo.getCategory().equals(Category.PRIMITIVE) && tableFieldTypeInfo - .equals(TypeInfoFactory.stringTypeInfo))) { + && tableFieldTypeInfo.getCategory().equals(Category.PRIMITIVE) && tableFieldTypeInfo + .equals(TypeInfoFactory.stringTypeInfo))) { // need to do some conversions here converted = true; if (tableFieldTypeInfo.getCategory() != Category.PRIMITIVE) { @@ -3607,12 +3657,12 @@ column = null; } else { column = TypeCheckProcFactory.DefaultExprProcessor - .getFuncExprNodeDesc(tableFieldTypeInfo.getTypeName(), - column); + .getFuncExprNodeDesc(tableFieldTypeInfo.getTypeName(), + column); } if (column == null) { String reason = "Cannot convert column " + i + " from " - + rowFieldTypeInfo + " to " + tableFieldTypeInfo + "."; + + rowFieldTypeInfo + " to " + tableFieldTypeInfo + "."; throw new SemanticException(ErrorMsg.TARGET_TABLE_COLUMN_MISMATCH .getMsg(qb.getParseInfo().getDestForClause(dest), reason)); } @@ -3624,7 +3674,7 @@ // deal with dynamic partition columns: convert ExprNodeDesc type to String?? if (dynPart && dpCtx != null && dpCtx.getNumDPCols() > 0) { // DP columns starts with tableFields.size() - for (int i = tableFields.size(); i < rowFields.size(); ++i ) { + for (int i = tableFields.size(); i < rowFields.size(); ++i) { TypeInfo rowFieldTypeInfo = rowFields.get(i).getType(); ExprNodeDesc column = new ExprNodeColumnDesc( rowFieldTypeInfo, rowFields.get(i).getInternalName(), "", false); @@ -3645,7 +3695,7 @@ } Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild( new SelectDesc(expressions, colName), new RowSchema(rowResolver - .getColumnInfos()), input), rowResolver); + .getColumnInfos()), input), rowResolver); return output; } else { @@ -3656,7 +3706,7 @@ @SuppressWarnings("nls") private Operator genLimitPlan(String dest, QB qb, Operator input, int limit) - throws SemanticException { + throws SemanticException { // A map-only job can be optimized - instead of converting it to a // map-reduce job, we can have another map // job to do the same to avoid the cost of sorting in the map-reduce phase. @@ -3669,9 +3719,10 @@ new LimitDesc(limit), new RowSchema(inputRR.getColumnInfos()), input), inputRR); - if (LOG.isDebugEnabled()) + if (LOG.isDebugEnabled()) { LOG.debug("Created LimitOperator Plan for clause: " + dest - + " row schema: " + inputRR.toString()); + + " row schema: " + inputRR.toString()); + } return limitMap; } @@ -3698,9 +3749,10 @@ throw new SemanticException(ErrorMsg.UDTF_LATERAL_VIEW.getMsg()); } - if (LOG.isDebugEnabled()) + if (LOG.isDebugEnabled()) { LOG.debug("Table alias: " + outputTableAlias + " Col aliases: " - + colAliases); + + colAliases); + } // Use the RowResolver from the input operator to generate a input // ObjectInspector that can be used to initialize the UDTF. Then, the @@ -3716,7 +3768,7 @@ for (int i = 0; i < inputCols.size(); i++) { colNames.add(inputCols.get(i).getInternalName()); colOIs[i] = TypeInfoUtils - .getStandardWritableObjectInspectorFromTypeInfo(inputCols.get(i) + .getStandardWritableObjectInspectorFromTypeInfo(inputCols.get(i) .getType()); } StructObjectInspector outputOI = genericUDTF.initialize(colOIs); @@ -3728,7 +3780,7 @@ if (numUdtfCols != numSuppliedAliases) { throw new SemanticException(ErrorMsg.UDTF_ALIAS_MISMATCH .getMsg("expected " + numUdtfCols + " aliases " + "but got " - + numSuppliedAliases)); + + numSuppliedAliases)); } // Generate the output column info's / row resolver using internal names. @@ -3784,11 +3836,11 @@ } private ArrayList getParitionColsFromBucketCols(String dest, QB qb, Table tab, - TableDesc table_desc, Operator input, boolean convert) - throws SemanticException { + TableDesc table_desc, Operator input, boolean convert) + throws SemanticException { RowResolver inputRR = opParseCtx.get(input).getRR(); List tabBucketCols = tab.getBucketCols(); - List tabCols = tab.getCols(); + List tabCols = tab.getCols(); // Partition by the bucketing column List posns = new ArrayList(); @@ -3807,12 +3859,13 @@ return genConvertCol(dest, qb, tab, table_desc, input, posns, convert); } - private ArrayList genConvertCol(String dest, QB qb, Table tab, TableDesc table_desc, Operator input, - List posns, boolean convert) throws SemanticException { + private ArrayList genConvertCol(String dest, QB qb, Table tab, + TableDesc table_desc, Operator input, + List posns, boolean convert) throws SemanticException { StructObjectInspector oi = null; try { Deserializer deserializer = table_desc.getDeserializerClass() - .newInstance(); + .newInstance(); deserializer.initialize(conf, table_desc.getProperties()); oi = (StructObjectInspector) deserializer.getObjectInspector(); } catch (Exception e) { @@ -3821,17 +3874,17 @@ List tableFields = oi.getAllStructFieldRefs(); ArrayList rowFields = opParseCtx.get(input).getRR() - .getColumnInfos(); + .getColumnInfos(); // Check column type int columnNumber = posns.size(); ArrayList expressions = new ArrayList(columnNumber); - for (Integer posn: posns) { + for (Integer posn : posns) { ObjectInspector tableFieldOI = tableFields.get(posn).getFieldObjectInspector(); TypeInfo tableFieldTypeInfo = TypeInfoUtils.getTypeInfoFromObjectInspector(tableFieldOI); TypeInfo rowFieldTypeInfo = rowFields.get(posn).getType(); ExprNodeDesc column = new ExprNodeColumnDesc(rowFieldTypeInfo, rowFields.get(posn).getInternalName(), - rowFields.get(posn).getTabAlias(), rowFields.get(posn).getIsVirtualCol()); + rowFields.get(posn).getTabAlias(), rowFields.get(posn).getIsVirtualCol()); if (convert && !tableFieldTypeInfo.equals(rowFieldTypeInfo)) { // need to do some conversions here @@ -3840,14 +3893,14 @@ column = null; } else { column = TypeCheckProcFactory.DefaultExprProcessor - .getFuncExprNodeDesc(tableFieldTypeInfo.getTypeName(), - column); + .getFuncExprNodeDesc(tableFieldTypeInfo.getTypeName(), + column); } if (column == null) { String reason = "Cannot convert column " + posn + " from " - + rowFieldTypeInfo + " to " + tableFieldTypeInfo + "."; + + rowFieldTypeInfo + " to " + tableFieldTypeInfo + "."; throw new SemanticException(ErrorMsg.TARGET_TABLE_COLUMN_MISMATCH - .getMsg(qb.getParseInfo().getDestForClause(dest), reason)); + .getMsg(qb.getParseInfo().getDestForClause(dest), reason)); } } expressions.add(column); @@ -3856,11 +3909,12 @@ return expressions; } - private ArrayList getSortCols(String dest, QB qb, Table tab, TableDesc table_desc, Operator input, boolean convert) - throws SemanticException { + private ArrayList getSortCols(String dest, QB qb, Table tab, TableDesc table_desc, + Operator input, boolean convert) + throws SemanticException { RowResolver inputRR = opParseCtx.get(input).getRR(); List tabSortCols = tab.getSortCols(); - List tabCols = tab.getCols(); + List tabCols = tab.getCols(); // Partition by the bucketing column List posns = new ArrayList(); @@ -3881,10 +3935,10 @@ @SuppressWarnings("nls") private Operator genReduceSinkPlanForSortingBucketing(Table tab, Operator input, - ArrayList sortCols, - ArrayList partitionCols, - int numReducers) - throws SemanticException { + ArrayList sortCols, + ArrayList partitionCols, + int numReducers) + throws SemanticException { RowResolver inputRR = opParseCtx.get(input).getRR(); // For the generation of the values expression just get the inputs @@ -3911,8 +3965,8 @@ Operator interim = putOpInsertMap(OperatorFactory.getAndMakeChild(PlanUtils .getReduceSinkDesc(sortCols, valueCols, outputColumns, false, -1, - partitionCols, order.toString(), numReducers), - new RowSchema(inputRR.getColumnInfos()), input), inputRR); + partitionCols, order.toString(), numReducers), + new RowSchema(inputRR.getColumnInfos()), input), inputRR); interim.setColumnExprMap(colExprMap); // Add the extract operator to get the value fields @@ -3922,19 +3976,20 @@ for (ColumnInfo colInfo : interim_rwsch.getColumnInfos()) { String[] info = interim_rwsch.reverseLookup(colInfo.getInternalName()); out_rwsch.put(info[0], info[1], new ColumnInfo( - getColumnInternalName(pos), colInfo.getType(), info[0], + getColumnInternalName(pos), colInfo.getType(), info[0], colInfo.getIsVirtualCol(), colInfo.isHiddenVirtualCol())); pos = Integer.valueOf(pos.intValue() + 1); } Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild( new ExtractDesc(new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, - Utilities.ReduceField.VALUE.toString(), "", false)), new RowSchema( - out_rwsch.getColumnInfos()), interim), out_rwsch); + Utilities.ReduceField.VALUE.toString(), "", false)), new RowSchema( + out_rwsch.getColumnInfos()), interim), out_rwsch); - if (LOG.isDebugEnabled()) + if (LOG.isDebugEnabled()) { LOG.debug("Created ReduceSink Plan for table: " + tab.getTableName() + - " row schema: " + out_rwsch.toString()); + " row schema: " + out_rwsch.toString()); + } return output; } @@ -3973,8 +4028,8 @@ // in strict mode, in the presence of order by, limit must be specified Integer limit = qb.getParseInfo().getDestLimit(dest); if (conf.getVar(HiveConf.ConfVars.HIVEMAPREDMODE).equalsIgnoreCase( - "strict") - && limit == null) { + "strict") + && limit == null) { throw new SemanticException(ErrorMsg.NO_LIMIT_WITH_ORDERBY .getMsg(sortExprs)); } @@ -4023,8 +4078,8 @@ } Operator interim = putOpInsertMap(OperatorFactory.getAndMakeChild(PlanUtils .getReduceSinkDesc(sortCols, valueCols, outputColumns, false, -1, - partitionCols, order.toString(), numReducers), - new RowSchema(inputRR.getColumnInfos()), input), inputRR); + partitionCols, order.toString(), numReducers), + new RowSchema(inputRR.getColumnInfos()), input), inputRR); interim.setColumnExprMap(colExprMap); // Add the extract operator to get the value fields @@ -4034,19 +4089,20 @@ for (ColumnInfo colInfo : interim_rwsch.getColumnInfos()) { String[] info = interim_rwsch.reverseLookup(colInfo.getInternalName()); out_rwsch.put(info[0], info[1], new ColumnInfo( - getColumnInternalName(pos), colInfo.getType(), info[0], + getColumnInternalName(pos), colInfo.getType(), info[0], colInfo.getIsVirtualCol(), colInfo.isHiddenVirtualCol())); pos = Integer.valueOf(pos.intValue() + 1); } Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild( new ExtractDesc(new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, - Utilities.ReduceField.VALUE.toString(), "", false)), new RowSchema( - out_rwsch.getColumnInfos()), interim), out_rwsch); + Utilities.ReduceField.VALUE.toString(), "", false)), new RowSchema( + out_rwsch.getColumnInfos()), interim), out_rwsch); - if (LOG.isDebugEnabled()) + if (LOG.isDebugEnabled()) { LOG.debug("Created ReduceSink Plan for clause: " + dest + " row schema: " - + out_rwsch.toString()); + + out_rwsch.toString()); + } return output; } @@ -4165,7 +4221,7 @@ .toString() + "." + col, valueInfo.getType(), src, valueInfo .getIsVirtualCol(), valueInfo.isHiddenVirtualCol()); - + colExprMap.put(newColInfo.getInternalName(), inputExpr); outputRS.put(src, field, newColInfo); } @@ -4180,16 +4236,16 @@ // Cartesian product is not supported in strict mode if (conf.getVar(HiveConf.ConfVars.HIVEMAPREDMODE).equalsIgnoreCase( - "strict")) { + "strict")) { throw new SemanticException(ErrorMsg.NO_CARTESIAN_PRODUCT.getMsg()); } } ReduceSinkOperator rsOp = (ReduceSinkOperator) putOpInsertMap( OperatorFactory.getAndMakeChild(PlanUtils.getReduceSinkDesc(reduceKeys, - reduceValues, outputColumns, false, joinTree.getNextTag(), - reduceKeys.size(), numReds), new RowSchema(outputRS - .getColumnInfos()), child), outputRS); + reduceValues, outputColumns, false, joinTree.getNextTag(), + reduceKeys.size(), numReds), new RowSchema(outputRS + .getColumnInfos()), child), outputRS); rsOp.setColumnExprMap(colExprMap); return rsOp; } @@ -4283,7 +4339,7 @@ // create selection operator Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild( new SelectDesc(colList, columnNames, false), new RowSchema(inputRR - .getColumnInfos()), input), inputRR); + .getColumnInfos()), input), inputRR); output.setColumnExprMap(input.getColumnExprMap()); return output; @@ -4297,10 +4353,10 @@ // key // "tab.col" Operator inputOperatorInfo, GroupByDesc.Mode mode) - throws SemanticException { + throws SemanticException { RowResolver groupByInputRowResolver = opParseCtx.get(inputOperatorInfo) - .getRR(); + .getRR(); RowResolver groupByOutputRowResolver = new RowResolver(); ArrayList groupByKeys = new ArrayList(); ArrayList outputColumnNames = new ArrayList(); @@ -4333,15 +4389,15 @@ // Generate group-by operator Operator op = putOpInsertMap(OperatorFactory.getAndMakeChild( new GroupByDesc(mode, outputColumnNames, groupByKeys, aggregations, - false), new RowSchema(groupByOutputRowResolver.getColumnInfos()), - inputOperatorInfo), groupByOutputRowResolver); + false), new RowSchema(groupByOutputRowResolver.getColumnInfos()), + inputOperatorInfo), groupByOutputRowResolver); op.setColumnExprMap(colExprMap); return op; } private void genJoinOperatorTypeCheck(Operator left, Operator[] right) - throws SemanticException { + throws SemanticException { // keys[i] -> ArrayList for the i-th join operator key list ArrayList> keys = new ArrayList>(); int keyLength = 0; @@ -4375,7 +4431,7 @@ keys.get(i).set( k, TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc( - commonType.getTypeName(), keys.get(i).get(k))); + commonType.getTypeName(), keys.get(i).get(k))); } } } @@ -4392,7 +4448,7 @@ } private Operator genJoinPlan(QB qb, HashMap map) - throws SemanticException { + throws SemanticException { QBJoinTree joinTree = qb.getQbJoinTree(); Operator joinOp = genJoinOperator(qb, joinTree, map); return joinOp; @@ -4433,7 +4489,7 @@ int numCh = hintTblNames.getChildCount(); for (int tblPos = 0; tblPos < numCh; tblPos++) { String tblName = ((ASTNode) hintTblNames.getChild(tblPos)).getText() - .toLowerCase(); + .toLowerCase(); if (!cols.contains(tblName)) { cols.add(tblName); } @@ -4445,7 +4501,7 @@ } private QBJoinTree genUniqueJoinTree(QB qb, ASTNode joinParseTree) - throws SemanticException { + throws SemanticException { QBJoinTree joinTree = new QBJoinTree(); joinTree.setNoOuterJoin(false); @@ -4473,7 +4529,7 @@ String tableName = unescapeIdentifier(child.getChild(0).getText()); String alias = child.getChildCount() == 1 ? tableName : unescapeIdentifier(child.getChild(child.getChildCount() - 1) - .getText().toLowerCase()); + .getText().toLowerCase()); if (i == 0) { leftAliases.add(alias); @@ -4512,7 +4568,7 @@ case HiveParser.TOK_SUBQUERY: throw new SemanticException( - "Subqueries are not supported in UNIQUEJOIN"); + "Subqueries are not supported in UNIQUEJOIN"); default: throw new SemanticException("Unexpected UNIQUEJOIN structure"); @@ -4537,7 +4593,7 @@ } private QBJoinTree genJoinTree(QB qb, ASTNode joinParseTree) - throws SemanticException { + throws SemanticException { QBJoinTree joinTree = new QBJoinTree(); JoinCond[] condn = new JoinCond[1]; @@ -4574,7 +4630,7 @@ String table_name = unescapeIdentifier(left.getChild(0).getText()); String alias = left.getChildCount() == 1 ? table_name : unescapeIdentifier(left.getChild(left.getChildCount() - 1) - .getText().toLowerCase()); + .getText().toLowerCase()); joinTree.setLeftAlias(alias); String[] leftAliases = new String[1]; leftAliases[0] = alias; @@ -4601,7 +4657,7 @@ String tableName = unescapeIdentifier(right.getChild(0).getText()); String alias = right.getChildCount() == 1 ? tableName : unescapeIdentifier(right.getChild(right.getChildCount() - 1) - .getText().toLowerCase()); + .getText().toLowerCase()); String[] rightAliases = new String[1]; rightAliases[0] = alias; joinTree.setRightAliases(rightAliases); @@ -4696,7 +4752,7 @@ String[] nodeRightAliases = node.getRightAliases(); String[] trgtRightAliases = target.getRightAliases(); String[] rightAliases = new String[nodeRightAliases.length - + trgtRightAliases.length]; + + trgtRightAliases.length]; for (int i = 0; i < trgtRightAliases.length; i++) { rightAliases[i] = trgtRightAliases[i]; @@ -4859,7 +4915,7 @@ } private Operator insertSelectAllPlanForGroupBy(String dest, Operator input) - throws SemanticException { + throws SemanticException { OpParseContext inputCtx = opParseCtx.get(input); RowResolver inputRR = inputCtx.getRR(); ArrayList columns = inputRR.getColumnInfos(); @@ -4873,7 +4929,7 @@ } Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild( new SelectDesc(colList, columnNames, true), new RowSchema(inputRR - .getColumnInfos()), input), inputRR); + .getColumnInfos()), input), inputRR); output.setColumnExprMap(input.getColumnExprMap()); return output; } @@ -4944,7 +5000,7 @@ } private Operator createCommonReduceSink(QB qb, Operator input) - throws SemanticException { + throws SemanticException { // Go over all the tables and extract the common distinct key List distExprs = getCommonDistinctExprs(qb, input); @@ -4969,7 +5025,7 @@ if (reduceSinkOutputRowResolver.getExpression(distn) == null) { outputColumnNames.add(getColumnInternalName(reduceKeys.size() - 1)); String field = Utilities.ReduceField.KEY.toString() + "." - + getColumnInternalName(reduceKeys.size() - 1); + + getColumnInternalName(reduceKeys.size() - 1); ColumnInfo colInfo = new ColumnInfo(field, reduceKeys.get( reduceKeys.size() - 1).getTypeInfo(), "", false); reduceSinkOutputRowResolver.putExpression(distn, colInfo); @@ -4988,7 +5044,7 @@ ExprNodeDesc grpByExprNode = genExprNodeDesc(grpbyExpr, inputRR); reduceValues.add(grpByExprNode); String field = Utilities.ReduceField.VALUE.toString() + "." - + getColumnInternalName(reduceValues.size() - 1); + + getColumnInternalName(reduceValues.size() - 1); ColumnInfo colInfo = new ColumnInfo(field, reduceValues.get( reduceValues.size() - 1).getTypeInfo(), "", false); reduceSinkOutputRowResolver.putExpression(grpbyExpr, colInfo); @@ -4998,7 +5054,7 @@ // For each aggregation HashMap aggregationTrees = qbp - .getAggregationExprsForClause(dest); + .getAggregationExprsForClause(dest); assert (aggregationTrees != null); for (Map.Entry entry : aggregationTrees.entrySet()) { @@ -5013,12 +5069,12 @@ ExprNodeDesc paraExprNode = genExprNodeDesc(paraExpr, inputRR); reduceValues.add(paraExprNode); String field = Utilities.ReduceField.VALUE.toString() + "." - + getColumnInternalName(reduceValues.size() - 1); + + getColumnInternalName(reduceValues.size() - 1); ColumnInfo colInfo = new ColumnInfo(field, reduceValues.get( reduceValues.size() - 1).getTypeInfo(), "", false); reduceSinkOutputRowResolver.putExpression(paraExpr, colInfo); outputColumnNames - .add(getColumnInternalName(reduceValues.size() - 1)); + .add(getColumnInternalName(reduceValues.size() - 1)); } } } @@ -5026,9 +5082,9 @@ ReduceSinkOperator rsOp = (ReduceSinkOperator) putOpInsertMap( OperatorFactory.getAndMakeChild(PlanUtils.getReduceSinkDesc(reduceKeys, - reduceValues, outputColumnNames, true, -1, reduceKeys.size(), -1), - new RowSchema(reduceSinkOutputRowResolver.getColumnInfos()), input), - reduceSinkOutputRowResolver); + reduceValues, outputColumnNames, true, -1, reduceKeys.size(), -1), + new RowSchema(reduceSinkOutputRowResolver.getColumnInfos()), input), + reduceSinkOutputRowResolver); rsOp.setColumnExprMap(colExprMap); return rsOp; @@ -5038,14 +5094,13 @@ private Operator genBodyPlan(QB qb, Operator input) throws SemanticException { QBParseInfo qbp = qb.getParseInfo(); - TreeSet ks = new TreeSet(qbp.getClauseNames()); // For multi-group by with the same distinct, we ignore all user hints // currently. It doesnt matter whether he has asked to do // map-side aggregation or not. Map side aggregation is turned off boolean optimizeMultiGroupBy = (getCommonDistinctExprs(qb, input) != null); - Operator curr = null; + Operator curr = input; // If there are multiple group-bys, map-side aggregation is turned off, // there are no filters @@ -5162,8 +5217,9 @@ } } - if (LOG.isDebugEnabled()) + if (LOG.isDebugEnabled()) { LOG.debug("Created Body Plan for Query Block " + qb.getId()); + } return curr; } @@ -5171,7 +5227,7 @@ @SuppressWarnings("nls") private Operator genUnionPlan(String unionalias, String leftalias, Operator leftOp, String rightalias, Operator rightOp) - throws SemanticException { + throws SemanticException { // Currently, the unions are not merged - each union has only 2 parents. So, // a n-way union will lead to (n-1) union operators. @@ -5231,7 +5287,7 @@ rightOp.setChildOperators(child); List> parent = leftOp - .getParentOperators(); + .getParentOperators(); parent.add(rightOp); UnionDesc uDesc = ((UnionOperator) leftOp).getConf(); @@ -5245,7 +5301,7 @@ leftOp.setChildOperators(child); List> parent = rightOp - .getParentOperators(); + .getParentOperators(); parent.add(leftOp); UnionDesc uDesc = ((UnionOperator) rightOp).getConf(); uDesc.setNumInputs(uDesc.getNumInputs() + 1); @@ -5256,7 +5312,7 @@ // Create a new union operator Operator unionforward = OperatorFactory - .getAndMakeChild(new UnionDesc(), new RowSchema(unionoutRR + .getAndMakeChild(new UnionDesc(), new RowSchema(unionoutRR .getColumnInfos())); // set union operator as child of each of leftOp and rightOp @@ -5314,7 +5370,7 @@ private ExprNodeDesc genSamplePredicate(TableSample ts, List bucketCols, boolean useBucketCols, String alias, RowResolver rwsch, QBMetaData qbm, ExprNodeDesc planExpr) - throws SemanticException { + throws SemanticException { ExprNodeDesc numeratorExpr = new ExprNodeConstantDesc( TypeInfoFactory.intTypeInfo, Integer.valueOf(ts.getNumerator() - 1)); @@ -5348,16 +5404,16 @@ assert (hashfnExpr != null); LOG.info("hashfnExpr = " + hashfnExpr); ExprNodeDesc andExpr = TypeCheckProcFactory.DefaultExprProcessor - .getFuncExprNodeDesc("&", hashfnExpr, intMaxExpr); + .getFuncExprNodeDesc("&", hashfnExpr, intMaxExpr); assert (andExpr != null); LOG.info("andExpr = " + andExpr); ExprNodeDesc modExpr = TypeCheckProcFactory.DefaultExprProcessor - .getFuncExprNodeDesc("%", andExpr, denominatorExpr); + .getFuncExprNodeDesc("%", andExpr, denominatorExpr); assert (modExpr != null); LOG.info("modExpr = " + modExpr); LOG.info("numeratorExpr = " + numeratorExpr); equalsExpr = TypeCheckProcFactory.DefaultExprProcessor - .getFuncExprNodeDesc("==", modExpr, numeratorExpr); + .getFuncExprNodeDesc("==", modExpr, numeratorExpr); LOG.info("equalsExpr = " + equalsExpr); assert (equalsExpr != null); } @@ -5382,14 +5438,14 @@ rwsch = new RowResolver(); try { StructObjectInspector rowObjectInspector = (StructObjectInspector) tab - .getDeserializer().getObjectInspector(); + .getDeserializer().getObjectInspector(); List fields = rowObjectInspector - .getAllStructFieldRefs(); + .getAllStructFieldRefs(); for (int i = 0; i < fields.size(); i++) { rwsch.put(alias, fields.get(i).getFieldName(), new ColumnInfo(fields .get(i).getFieldName(), TypeInfoUtils .getTypeInfoFromObjectInspector(fields.get(i) - .getFieldObjectInspector()), alias, false)); + .getFieldObjectInspector()), alias, false)); } } catch (SerDeException e) { throw new RuntimeException(e); @@ -5403,7 +5459,7 @@ rwsch.put(alias, part_col.getName(), new ColumnInfo(part_col.getName(), TypeInfoFactory.stringTypeInfo, alias, true)); } - + //put all virutal columns in RowResolver. Iterator vcs = VirtualColumn.registry.values().iterator(); //use a list for easy cumtomize @@ -5416,7 +5472,19 @@ } // Create the root of the operator tree - top = putOpInsertMap(OperatorFactory.get(new TableScanDesc(alias, vcList), + TableScanDesc tableScanDesc = new TableScanDesc(alias, vcList); + + // If it is an analyze command, fill the tablescandesc with appropriate information + if (qb.getParseInfo().isAnalyzeCommand()) { + tableScanDesc.setGatherStats(true); + List partColumns = new ArrayList(); + for (FieldSchema part_col : tab.getPartCols()) { + partColumns.add(part_col.getName()); + } + tableScanDesc.setPartColumns(partColumns); + } + + top = putOpInsertMap(OperatorFactory.get(tableScanDesc, new RowSchema(rwsch.getColumnInfos())), rwsch); // Add this to the list of top operators - we always start from a table @@ -5482,7 +5550,7 @@ // Check if input can be pruned ts - .setInputPruning((sampleExprs == null || sampleExprs.size() == 0 || colsEqual)); + .setInputPruning((sampleExprs == null || sampleExprs.size() == 0 || colsEqual)); // check if input pruning is enough if ((sampleExprs == null || sampleExprs.size() == 0 || colsEqual) @@ -5495,8 +5563,8 @@ colsEqual, alias, rwsch, qb.getMetaData(), null); tableOp = OperatorFactory.getAndMakeChild(new FilterDesc( samplePredicate, true, new sampleDesc(ts.getNumerator(), ts - .getDenominator(), tabBucketCols, true)), - new RowSchema(rwsch.getColumnInfos()), top); + .getDenominator(), tabBucketCols, true)), + new RowSchema(rwsch.getColumnInfos()), top); } else { // need to add filter // create tableOp to be filterDesc and set as child to 'top' @@ -5514,7 +5582,7 @@ // has the user explicitly asked not to sample this table String unSampleTblList = conf - .getVar(HiveConf.ConfVars.HIVETESTMODENOSAMPLE); + .getVar(HiveConf.ConfVars.HIVETESTMODENOSAMPLE); String[] unSampleTbls = unSampleTblList.split(","); boolean unsample = false; for (String unSampleTbl : unSampleTbls) { @@ -5534,10 +5602,10 @@ ExprNodeDesc samplePred = genSamplePredicate(tsSample, tab .getBucketCols(), true, alias, rwsch, qb.getMetaData(), null); tableOp = OperatorFactory - .getAndMakeChild(new FilterDesc(samplePred, true, + .getAndMakeChild(new FilterDesc(samplePred, true, new sampleDesc(tsSample.getNumerator(), tsSample - .getDenominator(), tab.getBucketCols(), true)), - new RowSchema(rwsch.getColumnInfos()), top); + .getDenominator(), tab.getBucketCols(), true)), + new RowSchema(rwsch.getColumnInfos()), top); LOG.info("No need for sample filter"); } else { // The table is not bucketed, add a dummy filter :: rand() @@ -5547,7 +5615,7 @@ qb.getParseInfo().setTabSample(alias, tsSample); LOG.info("Need sample filter"); ExprNodeDesc randFunc = TypeCheckProcFactory.DefaultExprProcessor - .getFuncExprNodeDesc("rand", new ExprNodeConstantDesc(Integer + .getFuncExprNodeDesc("rand", new ExprNodeConstantDesc(Integer .valueOf(460476415))); ExprNodeDesc samplePred = genSamplePredicate(tsSample, null, false, alias, rwsch, qb.getMetaData(), randFunc); @@ -5561,12 +5629,13 @@ Operator output = putOpInsertMap(tableOp, rwsch); - if (LOG.isDebugEnabled()) + if (LOG.isDebugEnabled()) { LOG.debug("Created Table Plan for " + alias + " " + tableOp.toString()); + } return output; } - + private Operator genPlan(QBExpr qbexpr) throws SemanticException { if (qbexpr.getOpcode() == QBExpr.Opcode.NULLOP) { return genPlan(qbexpr.getQB()); @@ -5583,7 +5652,6 @@ @SuppressWarnings("nls") public Operator genPlan(QB qb) throws SemanticException { - // First generate all the opInfos for the elements in the from clause HashMap aliasToOpInfo = new HashMap(); @@ -5603,7 +5671,6 @@ // For all the source tables that have a lateral view, attach the // appropriate operators to the TS genLateralViewPlans(aliasToOpInfo, qb); - Operator srcOpInfo = null; // process join @@ -5628,11 +5695,11 @@ // later we can extend this to the union all case as well srcOpInfo = aliasToOpInfo.values().iterator().next(); } - Operator bodyOpInfo = genBodyPlan(qb, srcOpInfo); - if (LOG.isDebugEnabled()) + if (LOG.isDebugEnabled()) { LOG.debug("Created Plan for Query Block " + qb.getId()); + } this.qb = qb; return bodyOpInfo; @@ -5650,9 +5717,9 @@ */ void genLateralViewPlans(HashMap aliasToOpInfo, QB qb) - throws SemanticException { + throws SemanticException { Map> aliasToLateralViews = qb.getParseInfo() - .getAliasToLateralViews(); + .getAliasToLateralViews(); for (Entry e : aliasToOpInfo.entrySet()) { String alias = e.getKey(); // See if the alias has a lateral view. If so, chain the lateral view @@ -5678,7 +5745,7 @@ String[] tabCol = source.reverseLookup(col.getInternalName()); lvForwardRR.put(tabCol[0], tabCol[1], col); } - + Operator lvForward = putOpInsertMap(OperatorFactory.getAndMakeChild( new LateralViewForwardDesc(), new RowSchema(lvForwardRR.getColumnInfos()), op), lvForwardRR); @@ -5689,10 +5756,10 @@ // Get the all path by making a select(*). RowResolver allPathRR = opParseCtx.get(lvForward).getRR(); - //Operator allPath = op; + // Operator allPath = op; Operator allPath = putOpInsertMap(OperatorFactory.getAndMakeChild( - new SelectDesc(true), new RowSchema(allPathRR.getColumnInfos()), - lvForward), allPathRR); + new SelectDesc(true), new RowSchema(allPathRR.getColumnInfos()), + lvForward), allPathRR); // Get the UDTF Path QB blankQb = new QB(null, null, false); Operator udtfPath = genSelectPlan((ASTNode) lateralViewTree @@ -5716,7 +5783,7 @@ // LVmerge.. in the above order Map colExprMap = new HashMap(); - int i=0; + int i = 0; for (ColumnInfo c : allPathRR.getColumnInfos()) { String internalName = getColumnInternalName(i); i++; @@ -5729,6 +5796,7 @@ .getAndMakeChild(new LateralViewJoinDesc(outputInternalColNames), new RowSchema(lateralViewRR.getColumnInfos()), allPath, udtfPath), lateralViewRR); + lateralViewJoin.setColumnExprMap(colExprMap); op = lateralViewJoin; } @@ -5782,7 +5850,7 @@ boolean noMapRed = false; Iterator> iter = qb.getMetaData() - .getAliasToTable().entrySet().iterator(); + .getAliasToTable().entrySet().iterator(); Table tab = (iter.next()).getValue(); if (!tab.isPartitioned()) { if (qbParseInfo.getDestToWhereExpr().isEmpty()) { @@ -5792,7 +5860,6 @@ inputs.add(new ReadEntity(tab)); } } else { - if (topOps.size() == 1) { TableScanOperator ts = (TableScanOperator) topOps.values().toArray()[0]; @@ -5842,7 +5909,7 @@ if (noMapRed) { if (fetch.getTblDesc() != null) { PlanUtils.configureTableJobPropertiesForStorageHandler( - fetch.getTblDesc()); + fetch.getTblDesc()); } fetchTask = (FetchTask) TaskFactory.get(fetch, conf); setFetchTask(fetchTask); @@ -5863,14 +5930,15 @@ fetch = new FetchWork(new Path(loadFileWork.get(0).getSourceDir()).toString(), new TableDesc(LazySimpleSerDe.class, - TextInputFormat.class, IgnoreKeyTextOutputFormat.class, Utilities - .makeProperties(SERIALIZATION_FORMAT, "" + Utilities.ctrlaCode, - LIST_COLUMNS, cols, - LIST_COLUMN_TYPES, colTypes)), - qb.getParseInfo().getOuterQueryLimit()); + TextInputFormat.class, IgnoreKeyTextOutputFormat.class, Utilities + .makeProperties(SERIALIZATION_FORMAT, "" + Utilities.ctrlaCode, + LIST_COLUMNS, cols, + LIST_COLUMN_TYPES, colTypes)), + qb.getParseInfo().getOuterQueryLimit()); fetchTask = (FetchTask) TaskFactory.get(fetch, conf); setFetchTask(fetchTask); + } else { new ArrayList(); for (LoadTableDesc ltd : loadTableWork) { @@ -6035,14 +6103,14 @@ } Operator reducer = ((MapredWork) task.getWork()) - .getReducer(); + .getReducer(); if (reducer != null) { LOG.info("Generating counters for operator " + reducer); generateCountersOperator(reducer); } } else if (task instanceof ConditionalTask) { List> listTasks = ((ConditionalTask) task) - .getListTasks(); + .getListTasks(); for (Task tsk : listTasks) { generateCountersTask(tsk); } @@ -6085,7 +6153,7 @@ } } else if (task instanceof ConditionalTask) { List> listTasks = ((ConditionalTask) task) - .getListTasks(); + .getListTasks(); for (Task tsk : listTasks) { breakTaskTree(tsk); } @@ -6122,7 +6190,7 @@ MapredWork work = (MapredWork) task.getWork(); work.deriveExplainAttributes(); HashMap> opMap = work - .getAliasToWork(); + .getAliasToWork(); if (!opMap.isEmpty()) { for (Operator op : opMap.values()) { GenMapRedUtils.setKeyAndValueDesc(work, op); @@ -6130,7 +6198,7 @@ } } else if (task instanceof ConditionalTask) { List> listTasks = ((ConditionalTask) task) - .getListTasks(); + .getListTasks(); for (Task tsk : listTasks) { setKeyDescTaskTree(tsk); } @@ -6192,12 +6260,12 @@ LOG.info("Completed getting MetaData in Semantic Analysis"); // Save the result schema derived from the sink operator produced - // by genPlan. This has the correct column names, which clients + // by genPlan. This has the correct column names, which clients // such as JDBC would prefer instead of the c0, c1 we'll end // up with later. + Operator sinkOp = genPlan(qb); - resultSchema = - convertRowSchemaToViewSchema(opParseCtx.get(sinkOp).getRR()); + resultSchema = convertRowSchemaToViewSchema(opParseCtx.get(sinkOp).getRR()); if (createVwDesc != null) { saveViewDefinition(); @@ -6242,7 +6310,7 @@ // Make a copy of the statement's result schema, since we may // modify it below as part of imposing view column names. List derivedSchema = - new ArrayList(resultSchema); + new ArrayList(resultSchema); validateColumnNameUniqueness(derivedSchema); List imposedSchema = createVwDesc.getSchema(); @@ -6327,7 +6395,7 @@ */ @SuppressWarnings("nls") public ExprNodeDesc genExprNodeDesc(ASTNode expr, RowResolver input) - throws SemanticException { + throws SemanticException { // We recursively create the exprNodeDesc. Base cases: when we encounter // a column ref, we convert that into an exprNodeColumnDesc; when we // encounter @@ -6361,7 +6429,7 @@ opRules.put(new RuleRegExp("R2", HiveParser.Number + "%"), TypeCheckProcFactory.getNumExprProcessor()); opRules - .put(new RuleRegExp("R3", HiveParser.Identifier + "%|" + .put(new RuleRegExp("R3", HiveParser.Identifier + "%|" + HiveParser.StringLiteral + "%|" + HiveParser.TOK_CHARSETLITERAL + "%|" + HiveParser.KW_IF + "%|" + HiveParser.KW_CASE + "%|" + HiveParser.KW_WHEN + "%|" + HiveParser.KW_IN + "%|" @@ -6438,7 +6506,7 @@ * @throws SemanticException */ static String getTabAliasForCol(QBMetaData qbm, String colName, ASTNode pt) - throws SemanticException { + throws SemanticException { String tabAlias = null; boolean found = false; @@ -6466,7 +6534,7 @@ } private void validate(Task task) - throws SemanticException { + throws SemanticException { if (task.getChildTasks() == null) { return; } @@ -6492,7 +6560,7 @@ * to the SerDe and Storage Format. */ private ASTNode analyzeCreateTable(ASTNode ast, QB qb) - throws SemanticException { + throws SemanticException { String tableName = unescapeIdentifier(ast.getChild(0).getText()); String likeTableName = null; List cols = new ArrayList(); @@ -6514,7 +6582,7 @@ RowFormatParams rowFormatParams = new RowFormatParams(); StorageFormat storageFormat = new StorageFormat(); AnalyzeCreateCommonVars shared = new AnalyzeCreateCommonVars(); - + LOG.info("Creating table " + tableName + " position=" + ast.getCharPositionInLine()); int numCh = ast.getChildCount(); @@ -6588,11 +6656,11 @@ bucketCols = getColumnNames((ASTNode) child.getChild(0)); if (child.getChildCount() == 2) { numBuckets = (Integer.valueOf(child.getChild(1).getText())) - .intValue(); + .intValue(); } else { sortCols = getColumnNamesOrder((ASTNode) child.getChild(1)); numBuckets = (Integer.valueOf(child.getChild(2).getText())) - .intValue(); + .intValue(); } break; case HiveParser.TOK_TABLEROWFORMAT: @@ -6608,8 +6676,7 @@ child = (ASTNode) child.getChild(0); shared.serde = unescapeSQLString(child.getChild(0).getText()); if (child.getChildCount() == 2) { - readProps((ASTNode) (child.getChild(1).getChild(0)), - shared.serdeProps); + readProps((ASTNode) (child.getChild(1).getChild(0)), shared.serdeProps); } break; default: @@ -6642,8 +6709,8 @@ case CREATE_TABLE: // REGULAR CREATE TABLE DDL crtTblDesc = new CreateTableDesc(tableName, isExt, cols, partCols, bucketCols, sortCols, numBuckets, rowFormatParams.fieldDelim, rowFormatParams.fieldEscape, - rowFormatParams.collItemDelim, rowFormatParams.mapKeyDelim, rowFormatParams.lineDelim, comment, - storageFormat.inputFormat, storageFormat.outputFormat, location, shared.serde, + rowFormatParams.collItemDelim, rowFormatParams.mapKeyDelim, rowFormatParams.lineDelim, comment, + storageFormat.inputFormat, storageFormat.outputFormat, location, shared.serde, storageFormat.storageHandler, shared.serdeProps, tblProps, ifNotExists); validateCreateTable(crtTblDesc); @@ -6688,7 +6755,7 @@ } private ASTNode analyzeCreateView(ASTNode ast, QB qb) - throws SemanticException { + throws SemanticException { String tableName = unescapeIdentifier(ast.getChild(0).getText()); List cols = null; boolean ifNotExists = false; @@ -6723,7 +6790,7 @@ } createVwDesc = new CreateViewDesc( - tableName, cols, comment, tblProps, ifNotExists); + tableName, cols, comment, tblProps, ifNotExists); unparseTranslator.enable(); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), createVwDesc), conf)); @@ -6756,7 +6823,7 @@ } private void validateCreateTable(CreateTableDesc crtTblDesc) - throws SemanticException { + throws SemanticException { if ((crtTblDesc.getCols() == null) || (crtTblDesc.getCols().size() == 0)) { // for now make sure that serde exists @@ -6770,12 +6837,12 @@ if (crtTblDesc.getStorageHandler() == null) { try { Class origin = Class.forName(crtTblDesc.getOutputFormat(), true, - JavaUtils.getClassLoader()); + JavaUtils.getClassLoader()); Class replaced = HiveFileFormatUtils - .getOutputFormatSubstitute(origin); + .getOutputFormatSubstitute(origin); if (replaced == null) { throw new SemanticException(ErrorMsg.INVALID_OUTPUT_FORMAT_TYPE - .getMsg()); + .getMsg()); } } catch (ClassNotFoundException e) { throw new SemanticException(ErrorMsg.INVALID_OUTPUT_FORMAT_TYPE.getMsg()); @@ -6842,23 +6909,25 @@ } private void decideExecMode(List> rootTasks, Context ctx) - throws SemanticException { + throws SemanticException { // bypass for explain queries for now - if (ctx.getExplain()) + if (ctx.getExplain()) { return; + } // user has told us to run in local mode or doesn't want auto-local mode if (ctx.isLocalOnlyExecutionMode() || - !conf.getBoolVar(HiveConf.ConfVars.LOCALMODEAUTO)) + !conf.getBoolVar(HiveConf.ConfVars.LOCALMODEAUTO)) { return; + } final Context lCtx = ctx; PathFilter p = new PathFilter () { - public boolean accept(Path file) { - return !lCtx.isMRTmpFileURI(file.toUri().getPath()); - } - }; + public boolean accept(Path file) { + return !lCtx.isMRTmpFileURI(file.toUri().getPath()); + } + }; List mrtasks = Utilities.getMRTasks(rootTasks); // map-reduce jobs will be run locally based on data size @@ -6867,13 +6936,13 @@ for (ExecDriver mrtask: mrtasks) { try { ContentSummary inputSummary = Utilities.getInputSummary - (ctx, (MapredWork)mrtask.getWork(), p); + (ctx, (MapredWork)mrtask.getWork(), p); int numReducers = getNumberOfReducers(mrtask.getWork(), conf); if (LOG.isDebugEnabled()) { - LOG.debug("Task: " + mrtask.getId() + ", Summary: " + - inputSummary.getLength() + "," + inputSummary.getFileCount() + "," - + numReducers); + LOG.debug("Task: " + mrtask.getId() + ", Summary: " + + inputSummary.getLength() + "," + inputSummary.getFileCount() + "," + + numReducers); } if(MapRedTask.isEligibleForLocalMode(conf, inputSummary, numReducers) != null) { @@ -6884,7 +6953,7 @@ throw new SemanticException (e); } } - + if(!hasNonLocalJob) { // none of the mapred tasks needs to be run locally. That means that the // query can be executed entirely in local mode. Save the current tracker @@ -6894,7 +6963,7 @@ console.printInfo("Automatically selecting local only mode for query"); // If all the tasks can be run locally, we can use local disk for - // storing intermediate data. + // storing intermediate data. /** * This code is commented out pending further testing/development @@ -6908,13 +6977,14 @@ * Make a best guess at trying to find the number of reducers */ private static int getNumberOfReducers(MapredWork mrwork, HiveConf conf) { - if (mrwork.getReducer() == null) + if (mrwork.getReducer() == null) { return 0; + } - if (mrwork.getNumReduceTasks() >= 0) + if (mrwork.getNumReduceTasks() >= 0) { return mrwork.getNumReduceTasks(); + } return conf.getIntVar(HiveConf.ConfVars.HADOOPNUMREDUCERS); } - } Index: ql/src/java/org/apache/hadoop/hive/ql/Driver.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java (revision 982011) +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java (working copy) @@ -66,6 +66,8 @@ import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; +import org.apache.hadoop.hive.ql.stats.StatsFactory; +import org.apache.hadoop.hive.ql.stats.StatsPublisher; import org.apache.hadoop.hive.serde2.ByteStream; import org.apache.hadoop.mapred.ClusterStatus; import org.apache.hadoop.mapred.JobClient; @@ -276,6 +278,12 @@ sem.analyze(tree, ctx); LOG.info("Semantic Analysis Completed"); + /* Initialize the temporary storage for gathering stats. + Possibly creating the schema if not exists. */ + if (ctx.isGatheringStats()) { + initTempStore(); + } + // validate the plan sem.validate(); @@ -292,7 +300,7 @@ if("true".equalsIgnoreCase(System.getProperty("test.serialize.qplan"))) { String queryPlanFileName = ctx.getLocalScratchDir(true) + Path.SEPARATOR_CHAR - + "queryplan.xml"; + + "queryplan.xml"; LOG.info("query plan = " + queryPlanFileName); queryPlanFileName = new Path(queryPlanFileName).toUri().getPath(); @@ -487,7 +495,7 @@ if (exitVal != 0) { // TODO: This error messaging is not very informative. Fix that. errorMessage = "FAILED: Execution Error, return code " + exitVal - + " from " + tsk.getClass().getName(); + + " from " + tsk.getClass().getName(); SQLState = "08S01"; console.printError(errorMessage); if (running.size() != 0) { @@ -722,7 +730,25 @@ } public org.apache.hadoop.hive.ql.plan.api.Query getQueryPlan() - throws IOException { + throws IOException { return plan.getQueryPlan(); } + + /** + * This method initializes the temporary storage for gathering stats, possibly + * creating the schema if not exists. + */ + private void initTempStore() { + try { + StatsPublisher statsPublisher; + String statsImplementationClass = HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS); + if (StatsFactory.setImplementation(statsImplementationClass, conf)) { + statsPublisher = StatsFactory.getStatsPublisher(); + statsPublisher.init(conf); + } + } + catch (Exception e) { + LOG.warn("Error during temporary storage initializations " + e); + } + } } Index: ql/src/gen-javabean/org/apache/hadoop/hive/ql/plan/api/StageType.java =================================================================== --- ql/src/gen-javabean/org/apache/hadoop/hive/ql/plan/api/StageType.java (revision 982011) +++ ql/src/gen-javabean/org/apache/hadoop/hive/ql/plan/api/StageType.java (working copy) @@ -6,13 +6,11 @@ package org.apache.hadoop.hive.ql.plan.api; -import java.util.Set; -import java.util.HashSet; -import java.util.Collections; -import org.apache.thrift.IntRangeSet; +import java.util.HashMap; import java.util.Map; -import java.util.HashMap; +import org.apache.thrift.IntRangeSet; + public class StageType { public static final int CONDITIONAL = 0; public static final int COPY = 1; @@ -23,17 +21,19 @@ public static final int FUNC = 6; public static final int MAPREDLOCAL = 7; public static final int MOVE = 8; + public static final int STATS = 9; public static final IntRangeSet VALID_VALUES = new IntRangeSet( - CONDITIONAL, - COPY, - DDL, - MAPRED, - EXPLAIN, - FETCH, - FUNC, - MAPREDLOCAL, - MOVE ); + CONDITIONAL, + COPY, + DDL, + MAPRED, + EXPLAIN, + FETCH, + FUNC, + MAPREDLOCAL, + MOVE, + STATS); public static final Map VALUES_TO_NAMES = new HashMap() {{ put(CONDITIONAL, "CONDITIONAL"); @@ -45,5 +45,6 @@ put(FUNC, "FUNC"); put(MAPREDLOCAL, "MAPREDLOCAL"); put(MOVE, "MOVE"); + put(STATS, "STATS"); }}; } Index: ql/build.xml =================================================================== --- ql/build.xml (revision 982011) +++ ql/build.xml (working copy) @@ -46,6 +46,7 @@ + @@ -60,7 +61,7 @@ + classpath="${build.dir.hive}/anttasks/hive-anttasks-${version}.jar:${hive.root}/lib/velocity-1.5.jar:${hive.root}/lib/commons-collections-3.2.1.jar:${hive.root}/lib/commons-lang-2.4.jar:${hive.root}/lib/derby.jar"/>