Index: ql/src/test/results/clientpositive/stats8.q.out =================================================================== --- ql/src/test/results/clientpositive/stats8.q.out (revision 1367905) +++ ql/src/test/results/clientpositive/stats8.q.out (working copy) @@ -77,9 +77,9 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-08',hr=11) +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=11) PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-08',hr=11) +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=11) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -89,15 +89,44 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: analyze_srcpart #### A masked pattern was here #### -PREHOOK: query: describe extended analyze_srcpart +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -107,12 +136,42 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Table Information +Database: default #### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 1 + numPartitions 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: explain analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=12) compute statistics PREHOOK: type: QUERY POSTHOOK: query: explain analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=12) compute statistics @@ -162,9 +221,9 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-08',hr=12) +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=12) PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-08',hr=12) +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=12) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -174,12 +233,41 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-04-08, 12] +Database: default +Table: analyze_srcpart #### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: explain analyze table analyze_srcpart PARTITION(ds='2008-04-09',hr=11) compute statistics PREHOOK: type: QUERY POSTHOOK: query: explain analyze table analyze_srcpart PARTITION(ds='2008-04-09',hr=11) compute statistics @@ -229,9 +317,9 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-09',hr=11) +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=11) PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-09',hr=11) +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=11) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -241,12 +329,41 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-04-09, 11] +Database: default +Table: analyze_srcpart #### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: explain analyze table analyze_srcpart PARTITION(ds='2008-04-09',hr=12) compute statistics PREHOOK: type: QUERY POSTHOOK: query: explain analyze table analyze_srcpart PARTITION(ds='2008-04-09',hr=12) compute statistics @@ -296,9 +413,9 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-09',hr=12) +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=12) PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-09',hr=12) +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=12) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -308,12 +425,41 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-04-09, 12] +Database: default +Table: analyze_srcpart #### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: explain analyze table analyze_srcpart PARTITION(ds, hr) compute statistics PREHOOK: type: QUERY POSTHOOK: query: explain analyze table analyze_srcpart PARTITION(ds, hr) compute statistics @@ -375,9 +521,9 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-08',hr=11) +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=11) PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-08',hr=11) +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=11) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -387,15 +533,44 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: analyze_srcpart #### A masked pattern was here #### -PREHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-08',hr=12) +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 2000 + rawDataSize 21248 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=12) PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-08',hr=12) +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=12) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -405,15 +580,44 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-04-08, 12] +Database: default +Table: analyze_srcpart #### A masked pattern was here #### -PREHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-09',hr=11) +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 0 + rawDataSize 0 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=11) PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-09',hr=11) +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=11) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -423,15 +627,44 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-04-09, 11] +Database: default +Table: analyze_srcpart #### A masked pattern was here #### -PREHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-09',hr=12) +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 0 + rawDataSize 0 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=12) PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-09',hr=12) +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=12) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -441,15 +674,44 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-04-09, 12] +Database: default +Table: analyze_srcpart #### A masked pattern was here #### -PREHOOK: query: describe extended analyze_srcpart +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 0 + rawDataSize 0 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -459,9 +721,39 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Table Information +Database: default #### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 4 + numPartitions 4 + numRows 2000 + rawDataSize 21248 + totalSize 23248 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 Index: ql/src/test/results/clientpositive/stats3.q.out =================================================================== --- ql/src/test/results/clientpositive/stats3.q.out (revision 1367905) +++ ql/src/test/results/clientpositive/stats3.q.out (working copy) @@ -11,12 +11,90 @@ POSTHOOK: query: create table hive_test_src ( col1 string ) stored as textfile POSTHOOK: type: CREATETABLE POSTHOOK: Output: default@hive_test_src +PREHOOK: query: explain extended +load data local inpath '../data/files/test.dat' overwrite into table hive_test_src +PREHOOK: type: LOAD +POSTHOOK: query: explain extended +load data local inpath '../data/files/test.dat' overwrite into table hive_test_src +POSTHOOK: type: LOAD +ABSTRACT SYNTAX TREE: + (TOK_LOAD '../data/files/test.dat' (TOK_TAB (TOK_TABNAME hive_test_src)) local overwrite) + +STAGE DEPENDENCIES: + Stage-0 is a root stage + Stage-1 depends on stages: Stage-0 + Stage-2 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-0 + Copy +#### A masked pattern was here #### + + Stage: Stage-1 + Move Operator + tables: + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns col1 + columns.types string +#### A masked pattern was here #### + name default.hive_test_src + serialization.ddl struct hive_test_src { string col1} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.hive_test_src +#### A masked pattern was here #### + + Stage: Stage-2 + Stats-Aggr Operator + + PREHOOK: query: load data local inpath '../data/files/test.dat' overwrite into table hive_test_src PREHOOK: type: LOAD PREHOOK: Output: default@hive_test_src POSTHOOK: query: load data local inpath '../data/files/test.dat' overwrite into table hive_test_src POSTHOOK: type: LOAD POSTHOOK: Output: default@hive_test_src +PREHOOK: query: desc formatted hive_test_src +PREHOOK: type: DESCTABLE +POSTHOOK: query: desc formatted hive_test_src +POSTHOOK: type: DESCTABLE +# col_name data_type comment + +col1 string None + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 1 + numPartitions 0 + numRows 0 + rawDataSize 0 + totalSize 11 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: create table hive_test_dst ( col1 string ) partitioned by ( pcol1 string , pcol2 string) stored as sequencefile PREHOOK: type: CREATETABLE POSTHOOK: query: create table hive_test_dst ( col1 string ) partitioned by ( pcol1 string , pcol2 string) stored as sequencefile @@ -56,6 +134,84 @@ #### A masked pattern was here #### POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] 6 +PREHOOK: query: explain +insert overwrite table hive_test_dst partition ( pCol1='test_part', pcol2='test_Part') select col1 from hive_test_src +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table hive_test_dst partition ( pCol1='test_part', pcol2='test_Part') select col1 from hive_test_src +POSTHOOK: type: QUERY +POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME hive_test_src))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME hive_test_dst) (TOK_PARTSPEC (TOK_PARTVAL pCol1 'test_part') (TOK_PARTVAL pcol2 'test_Part')))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL col1))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + hive_test_src + TableScan + alias: hive_test_src + Select Operator + expressions: + expr: col1 + type: string + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.hive_test_dst + + Stage: Stage-5 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + tables: + partition: + pcol1 test_part + pcol2 test_Part + replace: true + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.hive_test_dst + + Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.hive_test_dst + + PREHOOK: query: insert overwrite table hive_test_dst partition ( pCol1='test_part', pcol2='test_Part') select col1 from hive_test_src PREHOOK: type: QUERY PREHOOK: Input: default@hive_test_src @@ -117,17 +273,47 @@ #### A masked pattern was here #### POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] -PREHOOK: query: describe extended hive_test_dst +PREHOOK: query: describe formatted hive_test_dst PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended hive_test_dst +POSTHOOK: query: describe formatted hive_test_dst POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] -col1 string -pcol1 string -pcol2 string +# col_name data_type comment +col1 string None + +# Partition Information +# col_name data_type comment + +pcol1 string None +pcol2 string None + +# Detailed Table Information +Database: default #### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 1 + numPartitions 1 + numRows 6 + rawDataSize 6 + totalSize 171 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.SequenceFileInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: drop table hive_test_src PREHOOK: type: DROPTABLE PREHOOK: Input: default@hive_test_src Index: ql/src/test/results/clientpositive/stats13.q.out =================================================================== --- ql/src/test/results/clientpositive/stats13.q.out (revision 1367905) +++ ql/src/test/results/clientpositive/stats13.q.out (working copy) @@ -123,9 +123,9 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: desc extended analyze_srcpart +PREHOOK: query: desc formatted analyze_srcpart PREHOOK: type: DESCTABLE -POSTHOOK: query: desc extended analyze_srcpart +POSTHOOK: query: desc formatted analyze_srcpart POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -135,15 +135,45 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Table Information +Database: default #### A masked pattern was here #### -PREHOOK: query: desc extended analyze_srcpart partition (ds='2008-04-08', hr=11) +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 1 + numPartitions 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-08', hr=11) PREHOOK: type: DESCTABLE -POSTHOOK: query: desc extended analyze_srcpart partition (ds='2008-04-08', hr=11) +POSTHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-08', hr=11) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -153,15 +183,44 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: analyze_srcpart #### A masked pattern was here #### -PREHOOK: query: desc extended analyze_srcpart partition (ds='2008-04-08', hr=12) +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-08', hr=12) PREHOOK: type: DESCTABLE -POSTHOOK: query: desc extended analyze_srcpart partition (ds='2008-04-08', hr=12) +POSTHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-08', hr=12) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -171,15 +230,40 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-04-08, 12] +Database: default +Table: analyze_srcpart #### A masked pattern was here #### -PREHOOK: query: desc extended analyze_srcpart partition (ds='2008-04-09', hr=11) +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-09', hr=11) PREHOOK: type: DESCTABLE -POSTHOOK: query: desc extended analyze_srcpart partition (ds='2008-04-09', hr=11) +POSTHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-09', hr=11) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -189,15 +273,40 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-04-09, 11] +Database: default +Table: analyze_srcpart #### A masked pattern was here #### -PREHOOK: query: desc extended analyze_srcpart partition (ds='2008-04-09', hr=12) +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-09', hr=12) PREHOOK: type: DESCTABLE -POSTHOOK: query: desc extended analyze_srcpart partition (ds='2008-04-09', hr=12) +POSTHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-09', hr=12) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -207,12 +316,37 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-04-09, 12] +Database: default +Table: analyze_srcpart #### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: create table analyze_srcpart2 like analyze_srcpart PREHOOK: type: CREATETABLE POSTHOOK: query: create table analyze_srcpart2 like analyze_srcpart @@ -226,9 +360,9 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: desc extended analyze_srcpart2 +PREHOOK: query: desc formatted analyze_srcpart2 PREHOOK: type: DESCTABLE -POSTHOOK: query: desc extended analyze_srcpart2 +POSTHOOK: query: desc formatted analyze_srcpart2 POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -238,9 +372,34 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Table Information +Database: default #### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 Index: ql/src/test/results/clientpositive/stats5.q.out =================================================================== --- ql/src/test/results/clientpositive/stats5.q.out (revision 1367905) +++ ql/src/test/results/clientpositive/stats5.q.out (working copy) @@ -36,11 +36,37 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@analyze_src POSTHOOK: Output: default@analyze_src -PREHOOK: query: describe extended analyze_src +PREHOOK: query: describe formatted analyze_src PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended analyze_src +POSTHOOK: query: describe formatted analyze_src POSTHOOK: type: DESCTABLE -key string -value string +# col_name data_type comment +key string None +value string None + +# Detailed Table Information +Database: default #### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 1 + numPartitions 0 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 Index: ql/src/test/results/clientpositive/stats10.q.out =================================================================== --- ql/src/test/results/clientpositive/stats10.q.out (revision 1367905) +++ ql/src/test/results/clientpositive/stats10.q.out (working copy) @@ -430,9 +430,9 @@ POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: bucket3_1 PARTITION(ds=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: bucket3_1 PARTITION(ds=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: describe extended bucket3_1 partition (ds='1') +PREHOOK: query: describe formatted bucket3_1 partition (ds='1') PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended bucket3_1 partition (ds='1') +POSTHOOK: query: describe formatted bucket3_1 partition (ds='1') POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] @@ -440,14 +440,43 @@ POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: bucket3_1 PARTITION(ds=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: bucket3_1 PARTITION(ds=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -key int -value string -ds string +# col_name data_type comment +key int None +value string None + +# Partition Information +# col_name data_type comment + +ds string None + +# Detailed Partition Information +Partition Value: [1] +Database: default +Table: bucket3_1 #### A masked pattern was here #### -PREHOOK: query: describe extended bucket3_1 partition (ds='2') +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 2 + numRows 0 + rawDataSize 0 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 2 +Bucket Columns: [key] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted bucket3_1 partition (ds='2') PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended bucket3_1 partition (ds='2') +POSTHOOK: query: describe formatted bucket3_1 partition (ds='2') POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] @@ -455,14 +484,43 @@ POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: bucket3_1 PARTITION(ds=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: bucket3_1 PARTITION(ds=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -key int -value string -ds string +# col_name data_type comment +key int None +value string None + +# Partition Information +# col_name data_type comment + +ds string None + +# Detailed Partition Information +Partition Value: [2] +Database: default +Table: bucket3_1 #### A masked pattern was here #### -PREHOOK: query: describe extended bucket3_1 +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 2 + numRows 1000 + rawDataSize 10624 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 2 +Bucket Columns: [key] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted bucket3_1 PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended bucket3_1 +POSTHOOK: query: describe formatted bucket3_1 POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] @@ -470,8 +528,38 @@ POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: bucket3_1 PARTITION(ds=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: bucket3_1 PARTITION(ds=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -key int -value string -ds string +# col_name data_type comment +key int None +value string None + +# Partition Information +# col_name data_type comment + +ds string None + +# Detailed Table Information +Database: default #### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 4 + numPartitions 2 + numRows 1000 + rawDataSize 10624 + totalSize 11624 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 2 +Bucket Columns: [key] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 Index: ql/src/test/results/clientpositive/stats7.q.out =================================================================== --- ql/src/test/results/clientpositive/stats7.q.out (revision 1367905) +++ ql/src/test/results/clientpositive/stats7.q.out (working copy) @@ -81,9 +81,9 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-08',hr=11) +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=11) PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-08',hr=11) +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=11) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -93,15 +93,44 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: analyze_srcpart #### A masked pattern was here #### -PREHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-08',hr=12) +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 0 + rawDataSize 0 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=12) PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-08',hr=12) +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=12) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -111,15 +140,44 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-04-08, 12] +Database: default +Table: analyze_srcpart #### A masked pattern was here #### -PREHOOK: query: describe extended analyze_srcpart +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 1000 + rawDataSize 10624 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -129,9 +187,39 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Table Information +Database: default #### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 2 + numPartitions 2 + numRows 1000 + rawDataSize 10624 + totalSize 11624 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 Index: ql/src/test/results/clientpositive/stats2.q.out =================================================================== --- ql/src/test/results/clientpositive/stats2.q.out (revision 1367905) +++ ql/src/test/results/clientpositive/stats2.q.out (working copy) @@ -82,9 +82,9 @@ POSTHOOK: Lineage: analyze_t1 PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_t1 PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_t1 PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: desc extended analyze_t1 +PREHOOK: query: desc formatted analyze_t1 PREHOOK: type: DESCTABLE -POSTHOOK: query: desc extended analyze_t1 +POSTHOOK: query: desc formatted analyze_t1 POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_t1 PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_t1 PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -94,12 +94,37 @@ POSTHOOK: Lineage: analyze_t1 PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_t1 PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_t1 PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Table Information +Database: default #### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: explain analyze table analyze_t1 partition (ds, hr) compute statistics PREHOOK: type: QUERY POSTHOOK: query: explain analyze table analyze_t1 partition (ds, hr) compute statistics @@ -161,9 +186,9 @@ POSTHOOK: Lineage: analyze_t1 PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_t1 PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_t1 PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: describe extended analyze_t1 +PREHOOK: query: describe formatted analyze_t1 PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended analyze_t1 +POSTHOOK: query: describe formatted analyze_t1 POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_t1 PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_t1 PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -173,9 +198,39 @@ POSTHOOK: Lineage: analyze_t1 PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_t1 PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_t1 PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Table Information +Database: default #### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 4 + numPartitions 4 + numRows 2000 + rawDataSize 21248 + totalSize 23248 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 Index: ql/src/test/results/clientpositive/stats12.q.out =================================================================== --- ql/src/test/results/clientpositive/stats12.q.out (revision 1367905) +++ ql/src/test/results/clientpositive/stats12.q.out (working copy) @@ -164,9 +164,9 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: desc extended analyze_srcpart +PREHOOK: query: desc formatted analyze_srcpart PREHOOK: type: DESCTABLE -POSTHOOK: query: desc extended analyze_srcpart +POSTHOOK: query: desc formatted analyze_srcpart POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -176,15 +176,45 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Table Information +Database: default #### A masked pattern was here #### -PREHOOK: query: desc extended analyze_srcpart partition (ds='2008-04-08', hr=11) +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 2 + numPartitions 2 + numRows 1000 + rawDataSize 10624 + totalSize 11624 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-08', hr=11) PREHOOK: type: DESCTABLE -POSTHOOK: query: desc extended analyze_srcpart partition (ds='2008-04-08', hr=11) +POSTHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-08', hr=11) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -194,15 +224,44 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: analyze_srcpart #### A masked pattern was here #### -PREHOOK: query: desc extended analyze_srcpart partition (ds='2008-04-08', hr=12) +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 1000 + rawDataSize 10624 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-08', hr=12) PREHOOK: type: DESCTABLE -POSTHOOK: query: desc extended analyze_srcpart partition (ds='2008-04-08', hr=12) +POSTHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-08', hr=12) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -212,15 +271,44 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-04-08, 12] +Database: default +Table: analyze_srcpart #### A masked pattern was here #### -PREHOOK: query: desc extended analyze_srcpart partition (ds='2008-04-09', hr=11) +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 0 + rawDataSize 0 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-09', hr=11) PREHOOK: type: DESCTABLE -POSTHOOK: query: desc extended analyze_srcpart partition (ds='2008-04-09', hr=11) +POSTHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-09', hr=11) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -230,15 +318,40 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-04-09, 11] +Database: default +Table: analyze_srcpart #### A masked pattern was here #### -PREHOOK: query: desc extended analyze_srcpart partition (ds='2008-04-09', hr=12) +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-09', hr=12) PREHOOK: type: DESCTABLE -POSTHOOK: query: desc extended analyze_srcpart partition (ds='2008-04-09', hr=12) +POSTHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-09', hr=12) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -248,9 +361,34 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-04-09, 12] +Database: default +Table: analyze_srcpart #### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 Index: ql/src/test/results/clientpositive/stats9.q.out =================================================================== --- ql/src/test/results/clientpositive/stats9.q.out (revision 1367905) +++ ql/src/test/results/clientpositive/stats9.q.out (working copy) @@ -48,13 +48,39 @@ POSTHOOK: Output: default@analyze_srcbucket POSTHOOK: Lineage: analyze_srcbucket.key SIMPLE [(srcbucket)srcbucket.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: analyze_srcbucket.value SIMPLE [(srcbucket)srcbucket.FieldSchema(name:value, type:string, comment:null), ] -PREHOOK: query: describe extended analyze_srcbucket +PREHOOK: query: describe formatted analyze_srcbucket PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended analyze_srcbucket +POSTHOOK: query: describe formatted analyze_srcbucket POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcbucket.key SIMPLE [(srcbucket)srcbucket.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: analyze_srcbucket.value SIMPLE [(srcbucket)srcbucket.FieldSchema(name:value, type:string, comment:null), ] -key int -value string +# col_name data_type comment +key int None +value string None + +# Detailed Table Information +Database: default #### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 1 + numPartitions 0 + numRows 1000 + rawDataSize 10603 + totalSize 11603 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 2 +Bucket Columns: [key] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 Index: ql/src/test/results/clientpositive/stats4.q.out =================================================================== --- ql/src/test/results/clientpositive/stats4.q.out (revision 1367905) +++ ql/src/test/results/clientpositive/stats4.q.out (working copy) @@ -2273,9 +2273,9 @@ 400 val_400 2008-12-31 12 200 val_200 2008-12-31 12 97 val_97 2008-12-31 12 -PREHOOK: query: describe extended nzhang_part1 partition(ds='2008-04-08',hr=11) +PREHOOK: query: describe formatted nzhang_part1 partition(ds='2008-04-08',hr=11) PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended nzhang_part1 partition(ds='2008-04-08',hr=11) +POSTHOOK: query: describe formatted nzhang_part1 partition(ds='2008-04-08',hr=11) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -2285,15 +2285,44 @@ POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: nzhang_part1 #### A masked pattern was here #### -PREHOOK: query: describe extended nzhang_part1 partition(ds='2008-04-08',hr=12) +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted nzhang_part1 partition(ds='2008-04-08',hr=12) PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended nzhang_part1 partition(ds='2008-04-08',hr=12) +POSTHOOK: query: describe formatted nzhang_part1 partition(ds='2008-04-08',hr=12) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -2303,15 +2332,44 @@ POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-04-08, 12] +Database: default +Table: nzhang_part1 #### A masked pattern was here #### -PREHOOK: query: describe extended nzhang_part2 partition(ds='2008-12-31',hr=11) +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted nzhang_part2 partition(ds='2008-12-31',hr=11) PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended nzhang_part2 partition(ds='2008-12-31',hr=11) +POSTHOOK: query: describe formatted nzhang_part2 partition(ds='2008-12-31',hr=11) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -2321,15 +2379,44 @@ POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-12-31, 11] +Database: default +Table: nzhang_part2 #### A masked pattern was here #### -PREHOOK: query: describe extended nzhang_part2 partition(ds='2008-12-31',hr=12) +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted nzhang_part2 partition(ds='2008-12-31',hr=12) PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended nzhang_part2 partition(ds='2008-12-31',hr=12) +POSTHOOK: query: describe formatted nzhang_part2 partition(ds='2008-12-31',hr=12) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -2339,15 +2426,44 @@ POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-12-31, 12] +Database: default +Table: nzhang_part2 #### A masked pattern was here #### -PREHOOK: query: describe extended nzhang_part1 +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted nzhang_part1 PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended nzhang_part1 +POSTHOOK: query: describe formatted nzhang_part1 POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -2357,15 +2473,45 @@ POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Table Information +Database: default #### A masked pattern was here #### -PREHOOK: query: describe extended nzhang_part2 +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 2 + numPartitions 2 + numRows 1000 + rawDataSize 10624 + totalSize 11624 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted nzhang_part2 PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended nzhang_part2 +POSTHOOK: query: describe formatted nzhang_part2 POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: nzhang_part1 PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -2375,12 +2521,42 @@ POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: nzhang_part2 PARTITION(ds=2008-12-31,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Table Information +Database: default #### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 2 + numPartitions 2 + numRows 1000 + rawDataSize 10624 + totalSize 11624 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: drop table nzhang_part1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@nzhang_part1 Index: ql/src/test/results/clientpositive/stats6.q.out =================================================================== --- ql/src/test/results/clientpositive/stats6.q.out (revision 1367905) +++ ql/src/test/results/clientpositive/stats6.q.out (working copy) @@ -64,9 +64,9 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-08',hr=11) +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=11) PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-08',hr=11) +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=11) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -76,15 +76,44 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: analyze_srcpart #### A masked pattern was here #### -PREHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-08',hr=12) +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=12) PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-08',hr=12) +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=12) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -94,15 +123,44 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-04-08, 12] +Database: default +Table: analyze_srcpart #### A masked pattern was here #### -PREHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-09',hr=11) +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=11) PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-09',hr=11) +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=11) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -112,15 +170,40 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-04-09, 11] +Database: default +Table: analyze_srcpart #### A masked pattern was here #### -PREHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-09',hr=12) +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=12) PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended analyze_srcpart PARTITION(ds='2008-04-09',hr=12) +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=12) POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -130,15 +213,40 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-04-09, 12] +Database: default +Table: analyze_srcpart #### A masked pattern was here #### -PREHOOK: query: describe extended analyze_srcpart +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart PREHOOK: type: DESCTABLE -POSTHOOK: query: describe extended analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] @@ -148,9 +256,39 @@ POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -key string default -value string default -ds string -hr string +# col_name data_type comment +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Table Information +Database: default #### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 2 + numPartitions 2 + numRows 1000 + rawDataSize 10624 + totalSize 11624 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 Index: ql/src/test/results/clientpositive/stats1.q.out =================================================================== --- ql/src/test/results/clientpositive/stats1.q.out (revision 1367905) +++ ql/src/test/results/clientpositive/stats1.q.out (working copy) @@ -185,13 +185,89 @@ 66 val_66 98 val_98 tst1 500 -PREHOOK: query: DESCRIBE EXTENDED tmptable +PREHOOK: query: DESCRIBE FORMATTED tmptable PREHOOK: type: DESCTABLE -POSTHOOK: query: DESCRIBE EXTENDED tmptable +POSTHOOK: query: DESCRIBE FORMATTED tmptable POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: tmptable.key EXPRESSION [(src1)s2.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tmptable.value EXPRESSION [(src)s1.null, (src1)s2.FieldSchema(name:value, type:string, comment:default), ] -key string -value string +# col_name data_type comment +key string None +value string None + +# Detailed Table Information +Database: default #### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 2 + numPartitions 0 + numRows 26 + rawDataSize 199 + totalSize 225 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: -- Load a file into a existing table +-- Some stats (numFiles, totalSize) should be updated correctly +-- Some other stats (numRows, rawDataSize) is not updated correctly +load data local inpath '../data/files/srcbucket20.txt' INTO TABLE tmptable +PREHOOK: type: LOAD +PREHOOK: Output: default@tmptable +POSTHOOK: query: -- Load a file into a existing table +-- Some stats (numFiles, totalSize) should be updated correctly +-- Some other stats (numRows, rawDataSize) is not updated correctly +load data local inpath '../data/files/srcbucket20.txt' INTO TABLE tmptable +POSTHOOK: type: LOAD +POSTHOOK: Output: default@tmptable +POSTHOOK: Lineage: tmptable.key EXPRESSION [(src1)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmptable.value EXPRESSION [(src)s1.null, (src1)s2.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: DESCRIBE FORMATTED tmptable +PREHOOK: type: DESCTABLE +POSTHOOK: query: DESCRIBE FORMATTED tmptable +POSTHOOK: type: DESCTABLE +POSTHOOK: Lineage: tmptable.key EXPRESSION [(src1)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmptable.value EXPRESSION [(src)s1.null, (src1)s2.FieldSchema(name:value, type:string, comment:default), ] +# col_name data_type comment + +key string None +value string None + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 3 + numPartitions 0 + numRows 26 + rawDataSize 199 + totalSize 1583 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 Index: ql/src/test/results/clientpositive/stats11.q.out =================================================================== --- ql/src/test/results/clientpositive/stats11.q.out (revision 1367905) +++ ql/src/test/results/clientpositive/stats11.q.out (working copy) @@ -20,6 +20,41 @@ POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part (key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: default@srcbucket_mapjoin_part +PREHOOK: query: explain +load data local inpath '../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +PREHOOK: type: LOAD +POSTHOOK: query: explain +load data local inpath '../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +POSTHOOK: type: LOAD +ABSTRACT SYNTAX TREE: + (TOK_LOAD '../data/files/srcbucket20.txt' (TOK_TAB (TOK_TABNAME srcbucket_mapjoin_part) (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08'))) local) + +STAGE DEPENDENCIES: + Stage-0 is a root stage + Stage-1 depends on stages: Stage-0 + Stage-2 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-0 + Copy +#### A masked pattern was here #### + + Stage: Stage-1 + Move Operator + tables: + partition: + ds 2008-04-08 + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part + + Stage: Stage-2 + Stats-Aggr Operator + + PREHOOK: query: load data local inpath '../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') PREHOOK: type: LOAD PREHOOK: Output: default@srcbucket_mapjoin_part @@ -27,24 +62,176 @@ POSTHOOK: type: LOAD POSTHOOK: Output: default@srcbucket_mapjoin_part POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: query: desc formatted srcbucket_mapjoin_part partition(ds='2008-04-08') +PREHOOK: type: DESCTABLE +POSTHOOK: query: desc formatted srcbucket_mapjoin_part partition(ds='2008-04-08') +POSTHOOK: type: DESCTABLE +# col_name data_type comment + +key int None +value string None + +# Partition Information +# col_name data_type comment + +ds string None + +# Detailed Partition Information +Partition Value: [2008-04-08] +Database: default +Table: srcbucket_mapjoin_part +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 0 + rawDataSize 0 + totalSize 1358 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 4 +Bucket Columns: [key] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: load data local inpath '../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') PREHOOK: type: LOAD PREHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 POSTHOOK: query: load data local inpath '../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') POSTHOOK: type: LOAD POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: query: desc formatted srcbucket_mapjoin_part partition(ds='2008-04-08') +PREHOOK: type: DESCTABLE +POSTHOOK: query: desc formatted srcbucket_mapjoin_part partition(ds='2008-04-08') +POSTHOOK: type: DESCTABLE +# col_name data_type comment + +key int None +value string None + +# Partition Information +# col_name data_type comment + +ds string None + +# Detailed Partition Information +Partition Value: [2008-04-08] +Database: default +Table: srcbucket_mapjoin_part +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 2 + numRows 0 + rawDataSize 0 + totalSize 2750 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 4 +Bucket Columns: [key] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: load data local inpath '../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') PREHOOK: type: LOAD PREHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 POSTHOOK: query: load data local inpath '../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') POSTHOOK: type: LOAD POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: query: desc formatted srcbucket_mapjoin_part partition(ds='2008-04-08') +PREHOOK: type: DESCTABLE +POSTHOOK: query: desc formatted srcbucket_mapjoin_part partition(ds='2008-04-08') +POSTHOOK: type: DESCTABLE +# col_name data_type comment + +key int None +value string None + +# Partition Information +# col_name data_type comment + +ds string None + +# Detailed Partition Information +Partition Value: [2008-04-08] +Database: default +Table: srcbucket_mapjoin_part +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 3 + numRows 0 + rawDataSize 0 + totalSize 4200 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 4 +Bucket Columns: [key] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: load data local inpath '../data/files/srcbucket23.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') PREHOOK: type: LOAD PREHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 POSTHOOK: query: load data local inpath '../data/files/srcbucket23.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') POSTHOOK: type: LOAD POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: query: desc formatted srcbucket_mapjoin_part partition(ds='2008-04-08') +PREHOOK: type: DESCTABLE +POSTHOOK: query: desc formatted srcbucket_mapjoin_part partition(ds='2008-04-08') +POSTHOOK: type: DESCTABLE +# col_name data_type comment + +key int None +value string None + +# Partition Information +# col_name data_type comment + +ds string None + +# Detailed Partition Information +Partition Value: [2008-04-08] +Database: default +Table: srcbucket_mapjoin_part +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 4 + numRows 0 + rawDataSize 0 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 4 +Bucket Columns: [key] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part_2 (key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE PREHOOK: type: CREATETABLE POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part_2 (key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE @@ -212,9 +399,14 @@ columns.types int:string #### A masked pattern was here #### name default.srcbucket_mapjoin + numFiles 2 + numPartitions 0 + numRows 0 + rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2750 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -227,9 +419,14 @@ columns.types int:string #### A masked pattern was here #### name default.srcbucket_mapjoin + numFiles 2 + numPartitions 0 + numRows 0 + rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2750 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.srcbucket_mapjoin @@ -627,10 +824,15 @@ columns.types int:string #### A masked pattern was here #### name default.srcbucket_mapjoin_part + numFiles 4 + numPartitions 1 + numRows 0 partition_columns ds + rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -643,10 +845,15 @@ columns.types int:string #### A masked pattern was here #### name default.srcbucket_mapjoin_part + numFiles 4 + numPartitions 1 + numRows 0 partition_columns ds + rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.srcbucket_mapjoin_part Index: ql/src/test/queries/clientpositive/stats9.q =================================================================== --- ql/src/test/queries/clientpositive/stats9.q (revision 1367905) +++ ql/src/test/queries/clientpositive/stats9.q (working copy) @@ -6,4 +6,4 @@ explain analyze table analyze_srcbucket compute statistics; analyze table analyze_srcbucket compute statistics; -describe extended analyze_srcbucket; +describe formatted analyze_srcbucket; Index: ql/src/test/queries/clientpositive/stats10.q =================================================================== --- ql/src/test/queries/clientpositive/stats10.q (revision 1367905) +++ ql/src/test/queries/clientpositive/stats10.q (working copy) @@ -23,6 +23,6 @@ explain analyze table bucket3_1 partition (ds) compute statistics; analyze table bucket3_1 partition (ds) compute statistics; -describe extended bucket3_1 partition (ds='1'); -describe extended bucket3_1 partition (ds='2'); -describe extended bucket3_1; +describe formatted bucket3_1 partition (ds='1'); +describe formatted bucket3_1 partition (ds='2'); +describe formatted bucket3_1; Index: ql/src/test/queries/clientpositive/stats2.q =================================================================== --- ql/src/test/queries/clientpositive/stats2.q (revision 1367905) +++ ql/src/test/queries/clientpositive/stats2.q (working copy) @@ -12,10 +12,10 @@ insert overwrite table analyze_t1 partition (ds, hr) select * from srcpart where ds is not null; -desc extended analyze_t1; +desc formatted analyze_t1; explain analyze table analyze_t1 partition (ds, hr) compute statistics; analyze table analyze_t1 partition (ds, hr) compute statistics; -describe extended analyze_t1; +describe formatted analyze_t1; Index: ql/src/test/queries/clientpositive/stats12.q =================================================================== --- ql/src/test/queries/clientpositive/stats12.q (revision 1367905) +++ ql/src/test/queries/clientpositive/stats12.q (working copy) @@ -11,9 +11,9 @@ analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr) compute statistics; -desc extended analyze_srcpart; -desc extended analyze_srcpart partition (ds='2008-04-08', hr=11); -desc extended analyze_srcpart partition (ds='2008-04-08', hr=12); -desc extended analyze_srcpart partition (ds='2008-04-09', hr=11); -desc extended analyze_srcpart partition (ds='2008-04-09', hr=12); +desc formatted analyze_srcpart; +desc formatted analyze_srcpart partition (ds='2008-04-08', hr=11); +desc formatted analyze_srcpart partition (ds='2008-04-08', hr=12); +desc formatted analyze_srcpart partition (ds='2008-04-09', hr=11); +desc formatted analyze_srcpart partition (ds='2008-04-09', hr=12); Index: ql/src/test/queries/clientpositive/stats4.q =================================================================== --- ql/src/test/queries/clientpositive/stats4.q (revision 1367905) +++ ql/src/test/queries/clientpositive/stats4.q (working copy) @@ -28,13 +28,13 @@ select * from nzhang_part1 where ds is not null and hr is not null; select * from nzhang_part2 where ds is not null and hr is not null; -describe extended nzhang_part1 partition(ds='2008-04-08',hr=11); -describe extended nzhang_part1 partition(ds='2008-04-08',hr=12); -describe extended nzhang_part2 partition(ds='2008-12-31',hr=11); -describe extended nzhang_part2 partition(ds='2008-12-31',hr=12); +describe formatted nzhang_part1 partition(ds='2008-04-08',hr=11); +describe formatted nzhang_part1 partition(ds='2008-04-08',hr=12); +describe formatted nzhang_part2 partition(ds='2008-12-31',hr=11); +describe formatted nzhang_part2 partition(ds='2008-12-31',hr=12); -describe extended nzhang_part1; -describe extended nzhang_part2; +describe formatted nzhang_part1; +describe formatted nzhang_part2; drop table nzhang_part1; drop table nzhang_part2; Index: ql/src/test/queries/clientpositive/stats6.q =================================================================== --- ql/src/test/queries/clientpositive/stats6.q (revision 1367905) +++ ql/src/test/queries/clientpositive/stats6.q (working copy) @@ -9,9 +9,9 @@ analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics; analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=12) compute statistics; -describe extended analyze_srcpart PARTITION(ds='2008-04-08',hr=11); -describe extended analyze_srcpart PARTITION(ds='2008-04-08',hr=12); -describe extended analyze_srcpart PARTITION(ds='2008-04-09',hr=11); -describe extended analyze_srcpart PARTITION(ds='2008-04-09',hr=12); +describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=11); +describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=12); +describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=11); +describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=12); -describe extended analyze_srcpart; +describe formatted analyze_srcpart; Index: ql/src/test/queries/clientpositive/stats8.q =================================================================== --- ql/src/test/queries/clientpositive/stats8.q (revision 1367905) +++ ql/src/test/queries/clientpositive/stats8.q (working copy) @@ -8,26 +8,26 @@ explain analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics; analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics; -describe extended analyze_srcpart PARTITION(ds='2008-04-08',hr=11); -describe extended analyze_srcpart; +describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=11); +describe formatted analyze_srcpart; explain analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=12) compute statistics; analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=12) compute statistics; -describe extended analyze_srcpart PARTITION(ds='2008-04-08',hr=12); +describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=12); explain analyze table analyze_srcpart PARTITION(ds='2008-04-09',hr=11) compute statistics; analyze table analyze_srcpart PARTITION(ds='2008-04-09',hr=11) compute statistics; -describe extended analyze_srcpart PARTITION(ds='2008-04-09',hr=11); +describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=11); explain analyze table analyze_srcpart PARTITION(ds='2008-04-09',hr=12) compute statistics; analyze table analyze_srcpart PARTITION(ds='2008-04-09',hr=12) compute statistics; -describe extended analyze_srcpart PARTITION(ds='2008-04-09',hr=12); +describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=12); explain analyze table analyze_srcpart PARTITION(ds, hr) compute statistics; analyze table analyze_srcpart PARTITION(ds, hr) compute statistics; -describe extended analyze_srcpart PARTITION(ds='2008-04-08',hr=11); -describe extended analyze_srcpart PARTITION(ds='2008-04-08',hr=12); -describe extended analyze_srcpart PARTITION(ds='2008-04-09',hr=11); -describe extended analyze_srcpart PARTITION(ds='2008-04-09',hr=12); -describe extended analyze_srcpart; +describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=11); +describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=12); +describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=11); +describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=12); +describe formatted analyze_srcpart; Index: ql/src/test/queries/clientpositive/stats1.q =================================================================== --- ql/src/test/queries/clientpositive/stats1.q (revision 1367905) +++ ql/src/test/queries/clientpositive/stats1.q (working copy) @@ -21,4 +21,10 @@ SELECT * FROM tmptable x SORT BY x.key, x.value; -DESCRIBE EXTENDED tmptable; +DESCRIBE FORMATTED tmptable; + +-- Load a file into a existing table +-- Some stats (numFiles, totalSize) should be updated correctly +-- Some other stats (numRows, rawDataSize) is not updated correctly +load data local inpath '../data/files/srcbucket20.txt' INTO TABLE tmptable; +DESCRIBE FORMATTED tmptable; \ No newline at end of file Index: ql/src/test/queries/clientpositive/stats11.q =================================================================== --- ql/src/test/queries/clientpositive/stats11.q (revision 1367905) +++ ql/src/test/queries/clientpositive/stats11.q (working copy) @@ -6,10 +6,17 @@ load data local inpath '../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin; CREATE TABLE srcbucket_mapjoin_part (key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE; +explain load data local inpath '../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08'); +load data local inpath '../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08'); + +desc formatted srcbucket_mapjoin_part partition(ds='2008-04-08'); load data local inpath '../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08'); +desc formatted srcbucket_mapjoin_part partition(ds='2008-04-08'); load data local inpath '../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08'); +desc formatted srcbucket_mapjoin_part partition(ds='2008-04-08'); load data local inpath '../data/files/srcbucket23.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08'); +desc formatted srcbucket_mapjoin_part partition(ds='2008-04-08'); CREATE TABLE srcbucket_mapjoin_part_2 (key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE; load data local inpath '../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin_part_2 partition(ds='2008-04-08'); Index: ql/src/test/queries/clientpositive/stats3.q =================================================================== --- ql/src/test/queries/clientpositive/stats3.q (revision 1367905) +++ ql/src/test/queries/clientpositive/stats3.q (working copy) @@ -4,15 +4,22 @@ drop table hive_test_dst; create table hive_test_src ( col1 string ) stored as textfile ; +explain extended load data local inpath '../data/files/test.dat' overwrite into table hive_test_src ; +load data local inpath '../data/files/test.dat' overwrite into table hive_test_src ; + +desc formatted hive_test_src; + create table hive_test_dst ( col1 string ) partitioned by ( pcol1 string , pcol2 string) stored as sequencefile; insert overwrite table hive_test_dst partition ( pcol1='test_part', pCol2='test_Part') select col1 from hive_test_src ; select * from hive_test_dst where pcol1='test_part' and pcol2='test_Part'; select count(1) from hive_test_dst; +explain insert overwrite table hive_test_dst partition ( pCol1='test_part', pcol2='test_Part') select col1 from hive_test_src ; +insert overwrite table hive_test_dst partition ( pCol1='test_part', pcol2='test_Part') select col1 from hive_test_src ; select * from hive_test_dst where pcol1='test_part' and pcol2='test_part'; select count(1) from hive_test_dst; @@ -21,7 +28,7 @@ select * from hive_test_dst where pcol1='test_part' and pcol2='test_part'; select * from hive_test_dst where pcol1='test_Part'; -describe extended hive_test_dst; +describe formatted hive_test_dst; drop table hive_test_src; drop table hive_test_dst; Index: ql/src/test/queries/clientpositive/stats13.q =================================================================== --- ql/src/test/queries/clientpositive/stats13.q (revision 1367905) +++ ql/src/test/queries/clientpositive/stats13.q (working copy) @@ -11,12 +11,12 @@ analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics; -desc extended analyze_srcpart; -desc extended analyze_srcpart partition (ds='2008-04-08', hr=11); -desc extended analyze_srcpart partition (ds='2008-04-08', hr=12); -desc extended analyze_srcpart partition (ds='2008-04-09', hr=11); -desc extended analyze_srcpart partition (ds='2008-04-09', hr=12); +desc formatted analyze_srcpart; +desc formatted analyze_srcpart partition (ds='2008-04-08', hr=11); +desc formatted analyze_srcpart partition (ds='2008-04-08', hr=12); +desc formatted analyze_srcpart partition (ds='2008-04-09', hr=11); +desc formatted analyze_srcpart partition (ds='2008-04-09', hr=12); create table analyze_srcpart2 like analyze_srcpart; -desc extended analyze_srcpart2; +desc formatted analyze_srcpart2; Index: ql/src/test/queries/clientpositive/stats5.q =================================================================== --- ql/src/test/queries/clientpositive/stats5.q (revision 1367905) +++ ql/src/test/queries/clientpositive/stats5.q (working copy) @@ -7,4 +7,4 @@ analyze table analyze_src compute statistics; -describe extended analyze_src; +describe formatted analyze_src; Index: ql/src/test/queries/clientpositive/stats7.q =================================================================== --- ql/src/test/queries/clientpositive/stats7.q (revision 1367905) +++ ql/src/test/queries/clientpositive/stats7.q (working copy) @@ -10,7 +10,7 @@ analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr) compute statistics; -describe extended analyze_srcpart PARTITION(ds='2008-04-08',hr=11); -describe extended analyze_srcpart PARTITION(ds='2008-04-08',hr=12); +describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=11); +describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=12); -describe extended analyze_srcpart; +describe formatted analyze_srcpart; Index: ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java (revision 1367905) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java (working copy) @@ -34,7 +34,7 @@ private LoadTableDesc loadTableDesc; // same as MoveWork.loadTableDesc -- for FileSinkOperator private LoadFileDesc loadFileDesc; // same as MoveWork.loadFileDesc -- for FileSinkOperator private String aggKey; // aggregation key prefix - private boolean statsReliable; // are stats completely reliable + private boolean statsReliable; // are stats completely reliable private boolean noStatsAggregator = false; Index: ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java (revision 1367905) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java (working copy) @@ -34,6 +34,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.ql.exec.DependencyCollectionTask; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.exec.Utilities; @@ -42,9 +43,10 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.plan.CopyWork; +import org.apache.hadoop.hive.ql.plan.DependencyCollectionWork; import org.apache.hadoop.hive.ql.plan.LoadTableDesc; import org.apache.hadoop.hive.ql.plan.MoveWork; -import org.apache.hadoop.util.Shell; +import org.apache.hadoop.hive.ql.plan.StatsWork; /** * LoadSemanticAnalyzer. @@ -259,19 +261,27 @@ LoadTableDesc loadTableWork = new LoadTableDesc(fromURI.toString(), loadTmpPath, Utilities.getTableDesc(ts.tableHandle), partSpec, isOverWrite); + Task childTask = TaskFactory.get(new MoveWork(getInputs(), + getOutputs(), loadTableWork, null, true), conf); if (rTask != null) { - rTask.addDependentTask(TaskFactory.get(new MoveWork(getInputs(), - getOutputs(), loadTableWork, null, true), conf)); + rTask.addDependentTask(childTask); } else { - rTask = TaskFactory.get(new MoveWork(getInputs(), getOutputs(), - loadTableWork, null, true), conf); + rTask = childTask; } rootTasks.add(rTask); + if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEINDEXAUTOUPDATE)) { IndexUpdater indexUpdater = new IndexUpdater(loadTableWork, getInputs(), conf); try { List> indexUpdateTasks = indexUpdater.generateUpdateTasks(); + DependencyCollectionTask dependencyTask = null; + if (!indexUpdateTasks.isEmpty()) { + dependencyTask = + (DependencyCollectionTask) TaskFactory.get(new DependencyCollectionWork(), conf); + childTask = dependencyTask; + } + for (Task updateTask : indexUpdateTasks) { //LOAD DATA will either have a copy & move or just a move, we always want the update to be dependent on the move if (rTask.getChildren() == null || rTask.getChildren().size() == 0) { @@ -279,10 +289,23 @@ } else { ((Task)rTask.getChildren().get(0)).addDependentTask(updateTask); } + updateTask.addDependentTask(dependencyTask); } } catch (HiveException e) { console.printInfo("WARNING: could not auto-update stale indexes, indexes are not out of sync"); } } + + // The user asked for stats to be collected. + // Some stats like number of rows require a scan of the data + // However, some other stats, like number of files, do not require a complete scan + // Update the stats which do not require a complete scan. + if (conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { + StatsWork statDesc = new StatsWork(loadTableWork); + statDesc.setNoStatsAggregator(true); + statDesc.setStatsReliable(conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE)); + Task statTask = TaskFactory.get(statDesc, conf); + childTask.addDependentTask(statTask); + } } }