diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index d99cfdf74d57071183e9385b5a3f2c5335e4ce60..1c41cfbba548e95145df5cd79f48d8e68a584ece 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -33,8 +33,11 @@ import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.InvalidInputException; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; @@ -42,6 +45,9 @@ import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; +import org.apache.hive.common.util.HiveStringUtils; + +import com.google.common.collect.Lists; /** * Hive specific implementation of alter @@ -128,7 +134,10 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, if(MetaStoreUtils.isCascadeNeededInAlterTable(oldt, newt)) { List parts = msdb.getPartitions(dbname, name, -1); for (Partition part : parts) { + List oldCols = part.getSd().getCols(); part.getSd().setCols(newt.getSd().getCols()); + String oldPartName = Warehouse.makePartName(oldt.getPartitionKeys(), part.getValues()); + updatePartColumnStatsForAlterColumns(msdb, part, oldPartName, part.getValues(), oldCols, part); msdb.alterPartition(dbname, name, part.getValues(), part); } } else { @@ -209,6 +218,13 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, Path newPartLocPath = new Path(oldUri.getScheme(), oldUri.getAuthority(), newPath); altps.add(ObjectPair.create(part, part.getSd().getLocation())); part.getSd().setLocation(newPartLocPath.toString()); + String oldPartName = Warehouse.makePartName(oldt.getPartitionKeys(), part.getValues()); + try { + //existing partition column stats is no longer valid, remove them + msdb.deletePartitionColumnStatistics(dbname, name, oldPartName, part.getValues(), null); + } catch (InvalidInputException iie) { + throw new InvalidOperationException("Unable to update partition stats in table rename." + iie); + } msdb.alterPartition(dbname, name, part.getValues(), part); } } @@ -219,6 +235,7 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, // alterPartition() MetaStoreUtils.updateUnpartitionedTableStatsFast(db, newt, wh, false, true); } + updateTableColumnStatsForAlterTable(msdb, oldt, newt); // now finally call alter table msdb.alterTable(dbname, name, newt); // commit the changes @@ -311,6 +328,7 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String if (MetaStoreUtils.requireCalStats(hiveConf, oldPart, new_part, tbl)) { MetaStoreUtils.updatePartitionStatsFast(new_part, wh, false, true); } + updatePartColumnStats(msdb, dbname, name, new_part.getValues(), new_part); msdb.alterPartition(dbname, name, new_part.getValues(), new_part); } catch (InvalidObjectException e) { throw new InvalidOperationException("alter is not possible"); @@ -349,6 +367,15 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String // if the external partition is renamed, the file should not change if (tbl.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) { new_part.getSd().setLocation(oldPart.getSd().getLocation()); + String oldPartName = Warehouse.makePartName(tbl.getPartitionKeys(), oldPart.getValues()); + try { + //existing partition column stats is no longer valid, remove + msdb.deletePartitionColumnStatistics(dbname, name, oldPartName, oldPart.getValues(), null); + } catch (NoSuchObjectException nsoe) { + //ignore + } catch (InvalidInputException iie) { + throw new InvalidOperationException("Unable to update partition stats in table rename." + iie); + } msdb.alterPartition(dbname, name, part_vals, new_part); } else { try { @@ -395,6 +422,15 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String if (MetaStoreUtils.requireCalStats(hiveConf, oldPart, new_part, tbl)) { MetaStoreUtils.updatePartitionStatsFast(new_part, wh, false, true); } + String oldPartName = Warehouse.makePartName(tbl.getPartitionKeys(), oldPart.getValues()); + try { + //existing partition column stats is no longer valid, remove + msdb.deletePartitionColumnStatistics(dbname, name, oldPartName, oldPart.getValues(), null); + } catch (NoSuchObjectException nsoe) { + //ignore + } catch (InvalidInputException iie) { + throw new InvalidOperationException("Unable to update partition stats in table rename." + iie); + } msdb.alterPartition(dbname, name, part_vals, new_part); } } @@ -461,6 +497,7 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String if (MetaStoreUtils.requireCalStats(hiveConf, oldTmpPart, tmpPart, tbl)) { MetaStoreUtils.updatePartitionStatsFast(tmpPart, wh, false, true); } + updatePartColumnStats(msdb, dbname, name, oldTmpPart.getValues(), tmpPart); } msdb.alterPartitions(dbname, name, partValsList, new_parts); } catch (InvalidObjectException e) { @@ -508,4 +545,126 @@ private Path constructRenamedPath(Path defaultNewPath, Path currentPath) { return new Path(currentUri.getScheme(), currentUri.getAuthority(), defaultNewPath.toUri().getPath()); } + + private void updatePartColumnStatsForAlterColumns(RawStore msdb, Partition oldPartition, + String oldPartName, List partVals, List oldCols, Partition newPart) + throws MetaException, InvalidObjectException { + String dbName = oldPartition.getDbName(); + String tableName = oldPartition.getTableName(); + try { + List oldPartNames = Lists.newArrayList(oldPartName); + List oldColNames = new ArrayList(oldCols.size()); + for (FieldSchema oldCol : oldCols) { + oldColNames.add(oldCol.getName()); + } + List newCols = newPart.getSd().getCols(); + List partsColStats = msdb.getPartitionColumnStatistics(dbName, tableName, + oldPartNames, oldColNames); + assert (partsColStats.size() <= 1); + for (ColumnStatistics partColStats : partsColStats) { //actually only at most one loop + List statsObjs = partColStats.getStatsObj(); + for (ColumnStatisticsObj statsObj : statsObjs) { + boolean found =false; + for (FieldSchema newCol : newCols) { + if (statsObj.getColName().equals(newCol.getName()) + && statsObj.getColType().equals(newCol.getType())) { + found = true; + break; + } + } + if (!found) { + msdb.deletePartitionColumnStatistics(dbName, tableName, oldPartName, partVals, + statsObj.getColName()); + } + } + } + } catch (NoSuchObjectException nsoe) { + LOG.debug("Could not find db entry." + nsoe); + //ignore + } catch (InvalidInputException iie) { + throw new InvalidObjectException + ("Invalid input to update partition column stats in alter table change columns" + iie); + } + } + + private void updatePartColumnStats(RawStore msdb, String dbName, String tableName, + List partVals, Partition newPart) throws MetaException, InvalidObjectException { + dbName = HiveStringUtils.normalizeIdentifier(dbName); + tableName = HiveStringUtils.normalizeIdentifier(tableName); + String newDbName = HiveStringUtils.normalizeIdentifier(newPart.getDbName()); + String newTableName = HiveStringUtils.normalizeIdentifier(newPart.getTableName()); + + Table oldTable = msdb.getTable(dbName, tableName); + if (oldTable == null) { + return; + } + + try { + Partition oldPartition = msdb.getPartition(dbName, tableName, partVals); + String oldPartName = Warehouse.makePartName(oldTable.getPartitionKeys(), partVals); + String newPartName = Warehouse.makePartName(oldTable.getPartitionKeys(), newPart.getValues()); + if (!dbName.equals(newDbName) || !tableName.equals(newTableName) + || !oldPartName.equals(newPartName)) { + msdb.deletePartitionColumnStatistics(dbName, tableName, oldPartName, partVals, null); + } else { + List oldCols = oldPartition.getSd().getCols(); + if (!MetaStoreUtils.areSameColumns(oldCols, newPart.getSd().getCols())) { + updatePartColumnStatsForAlterColumns(msdb, oldPartition, oldPartName, partVals, oldCols, newPart); + } + } + } catch (NoSuchObjectException nsoe) { + LOG.debug("Could not find db entry." + nsoe); + //ignore + } catch (InvalidInputException iie) { + throw new InvalidObjectException("Invalid input to update partition column stats." + iie); + } + } + + private void updateTableColumnStatsForAlterTable(RawStore msdb, Table oldTable, Table newTable) + throws MetaException, InvalidObjectException { + String dbName = oldTable.getDbName(); + String tableName = oldTable.getTableName(); + String newDbName = HiveStringUtils.normalizeIdentifier(newTable.getDbName()); + String newTableName = HiveStringUtils.normalizeIdentifier(newTable.getTableName()); + + try { + if (!dbName.equals(newDbName) || !tableName.equals(newTableName)) { + msdb.deleteTableColumnStatistics(dbName, tableName, null); + } else { + List oldCols = oldTable.getSd().getCols(); + List newCols = newTable.getSd().getCols(); + if (!MetaStoreUtils.areSameColumns(oldCols, newCols)) { + List oldColNames = new ArrayList(oldCols.size()); + for (FieldSchema oldCol : oldCols) { + oldColNames.add(oldCol.getName()); + } + + ColumnStatistics cs = msdb.getTableColumnStatistics(dbName, tableName, oldColNames); + if (cs == null) { + return; + } + + List statsObjs = cs.getStatsObj(); + for (ColumnStatisticsObj statsObj : statsObjs) { + boolean found = false; + for (FieldSchema newCol : newCols) { + if (statsObj.getColName().equalsIgnoreCase(newCol.getName()) + && statsObj.getColType().equals(newCol.getType())) { + found = true; + break; + } + } + if (!found) { + msdb.deleteTableColumnStatistics(dbName, tableName, statsObj.getColName()); + } + } + } + } + } catch (NoSuchObjectException nsoe) { + LOG.debug("Could not find db entry." + nsoe); + } catch (InvalidInputException e) { + //should not happen since the input were verified before passed in + throw new InvalidObjectException("Invalid inputs to update table column stats: " + e); + } + } } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index 612f927d5515fbd5c3257a04b85f9bcc4c6891f3..2c8cc1fea6ed99335a54e0b35300d47124b8fef0 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -559,23 +559,26 @@ static void throwExceptionIfIncompatibleColTypeChange( } static boolean isCascadeNeededInAlterTable(Table oldTable, Table newTable) { + //currently cascade only supports add/replace columns and + //changing column type/position/name/comments List oldCols = oldTable.getSd().getCols(); List newCols = newTable.getSd().getCols(); + return !areSameColumns(oldCols, newCols); + } - //currently cascade only supports add/replace columns and - //changing column type/position/name/comments + static boolean areSameColumns(List oldCols, List newCols) { if (oldCols.size() != newCols.size()) { - return true; + return false; } else { for (int i = 0; i < oldCols.size(); i++) { FieldSchema oldCol = oldCols.get(i); FieldSchema newCol = newCols.get(i); if(!oldCol.equals(newCol)) { - return true; + return false; } } } - return false; + return true; } /** diff --git a/ql/src/test/queries/clientpositive/alter_table_invalidate_column_stats.q b/ql/src/test/queries/clientpositive/alter_table_invalidate_column_stats.q new file mode 100644 index 0000000000000000000000000000000000000000..8104c8ca092aace4ee55c520c416b3a6ac8c6a15 --- /dev/null +++ b/ql/src/test/queries/clientpositive/alter_table_invalidate_column_stats.q @@ -0,0 +1,152 @@ +set hive.metastore.try.direct.sql=true; + +drop database if exists statsdb1; +create database statsdb1; +drop database if exists statsdb2; +create database statsdb2; + +create table statsdb1.testtable1 (col1 int, col2 string, col3 string); +insert into statsdb1.testtable1 select key, value, 'val3' from src limit 10; + +create table statsdb1.testpart1 (col1 int, col2 string, col3 string) partitioned by (part string); +insert into statsdb1.testpart1 partition (part = 'part1') select key, value, 'val3' from src limit 10; +insert into statsdb1.testpart1 partition (part = 'part2') select key, value, 'val3' from src limit 20; + +use statsdb1; + +analyze table testtable1 compute statistics for columns; + +describe formatted statsdb1.testtable1 col1; +describe formatted statsdb1.testtable1 col2; +describe formatted statsdb1.testtable1 col3; + +alter table testtable1 replace columns (col1 int, col2 string, col4 string); +describe formatted statsdb1.testtable1 col1; +describe formatted statsdb1.testtable1 col2; +describe formatted statsdb1.testtable1 col4; + +alter table testtable1 change col1 col1 string; +describe formatted statsdb1.testtable1 col1; +describe formatted statsdb1.testtable1 col2; +describe formatted statsdb1.testtable1 col4; + +alter table statsdb1.testtable1 rename to statsdb2.testtable2; + + +analyze table testpart1 compute statistics for columns; + +describe formatted statsdb1.testpart1 col1 partition (part = 'part1'); +describe formatted statsdb1.testpart1 col2 partition (part = 'part1'); +describe formatted statsdb1.testpart1 col3 partition (part = 'part1'); +describe formatted statsdb1.testpart1 col1 partition (part = 'part2'); +describe formatted statsdb1.testpart1 col2 partition (part = 'part2'); +describe formatted statsdb1.testpart1 col3 partition (part = 'part2'); + +alter table statsdb1.testpart1 partition (part = 'part2') rename to partition (part = 'part3'); +describe formatted statsdb1.testpart1 col1 partition (part = 'part1'); +describe formatted statsdb1.testpart1 col2 partition (part = 'part1'); +describe formatted statsdb1.testpart1 col3 partition (part = 'part1'); +describe formatted statsdb1.testpart1 col1 partition (part = 'part3'); +describe formatted statsdb1.testpart1 col2 partition (part = 'part3'); +describe formatted statsdb1.testpart1 col3 partition (part = 'part3'); + +alter table statsdb1.testpart1 replace columns (col1 int, col2 string, col4 string) cascade; +describe formatted statsdb1.testpart1 col1 partition (part = 'part1'); +describe formatted statsdb1.testpart1 col2 partition (part = 'part1'); +describe formatted statsdb1.testpart1 col4 partition (part = 'part1'); + +alter table statsdb1.testpart1 change column col1 col1 string; +set hive.exec.dynamic.partition = true; +alter table statsdb1.testpart1 partition (part) change column col1 col1 string; +describe formatted statsdb1.testpart1 col1 partition (part = 'part1'); +describe formatted statsdb1.testpart1 col2 partition (part = 'part1'); +describe formatted statsdb1.testpart1 col4 partition (part = 'part1'); + +alter table statsdb1.testpart1 rename to statsdb2.testpart2; +use statsdb2; + +alter table statsdb2.testpart2 drop partition (part = 'part1'); +drop table statsdb2.testpart2; + +drop table statsdb2.testtable2; + +use default; +drop database statsdb1; +drop database statsdb2; + + +set hive.metastore.try.direct.sql=false; + +drop database if exists statsdb1; +create database statsdb1; +drop database if exists statsdb2; +create database statsdb2; + +create table statsdb1.testtable1 (col1 int, col2 string, col3 string); +insert into statsdb1.testtable1 select key, value, 'val3' from src limit 10; + +create table statsdb1.testpart1 (col1 int, col2 string, col3 string) partitioned by (part string); +insert into statsdb1.testpart1 partition (part = 'part1') select key, value, 'val3' from src limit 10; +insert into statsdb1.testpart1 partition (part = 'part2') select key, value, 'val3' from src limit 20; + +use statsdb1; + +analyze table testtable1 compute statistics for columns; + +describe formatted statsdb1.testtable1 col1; +describe formatted statsdb1.testtable1 col2; +describe formatted statsdb1.testtable1 col3; + +alter table testtable1 replace columns (col1 int, col2 string, col4 string); +describe formatted statsdb1.testtable1 col1; +describe formatted statsdb1.testtable1 col2; +describe formatted statsdb1.testtable1 col4; + +alter table testtable1 change col1 col1 string; +describe formatted statsdb1.testtable1 col1; +describe formatted statsdb1.testtable1 col2; +describe formatted statsdb1.testtable1 col4; + +alter table statsdb1.testtable1 rename to statsdb2.testtable2; + + +analyze table testpart1 compute statistics for columns; + +describe formatted statsdb1.testpart1 col1 partition (part = 'part1'); +describe formatted statsdb1.testpart1 col2 partition (part = 'part1'); +describe formatted statsdb1.testpart1 col3 partition (part = 'part1'); +describe formatted statsdb1.testpart1 col1 partition (part = 'part2'); +describe formatted statsdb1.testpart1 col2 partition (part = 'part2'); +describe formatted statsdb1.testpart1 col3 partition (part = 'part2'); + +alter table statsdb1.testpart1 partition (part = 'part2') rename to partition (part = 'part3'); +describe formatted statsdb1.testpart1 col1 partition (part = 'part1'); +describe formatted statsdb1.testpart1 col2 partition (part = 'part1'); +describe formatted statsdb1.testpart1 col3 partition (part = 'part1'); +describe formatted statsdb1.testpart1 col1 partition (part = 'part3'); +describe formatted statsdb1.testpart1 col2 partition (part = 'part3'); +describe formatted statsdb1.testpart1 col3 partition (part = 'part3'); + +alter table statsdb1.testpart1 replace columns (col1 int, col2 string, col4 string) cascade; +describe formatted statsdb1.testpart1 col1 partition (part = 'part1'); +describe formatted statsdb1.testpart1 col2 partition (part = 'part1'); +describe formatted statsdb1.testpart1 col4 partition (part = 'part1'); + +alter table statsdb1.testpart1 change column col1 col1 string; +set hive.exec.dynamic.partition = true; +alter table statsdb1.testpart1 partition (part) change column col1 col1 string; +describe formatted statsdb1.testpart1 col1 partition (part = 'part1'); +describe formatted statsdb1.testpart1 col2 partition (part = 'part1'); +describe formatted statsdb1.testpart1 col4 partition (part = 'part1'); + +alter table statsdb1.testpart1 rename to statsdb2.testpart2; +use statsdb2; + +alter table statsdb2.testpart2 drop partition (part = 'part1'); +drop table statsdb2.testpart2; + +drop table statsdb2.testtable2; + +use default; +drop database statsdb1; +drop database statsdb2; \ No newline at end of file diff --git a/ql/src/test/results/clientpositive/alter_table_invalidate_column_stats.q.out b/ql/src/test/results/clientpositive/alter_table_invalidate_column_stats.q.out new file mode 100644 index 0000000000000000000000000000000000000000..6ae4f253cd30103d73ad5d25d51021d769a6b3da --- /dev/null +++ b/ql/src/test/results/clientpositive/alter_table_invalidate_column_stats.q.out @@ -0,0 +1,932 @@ +PREHOOK: query: drop database if exists statsdb1 +PREHOOK: type: DROPDATABASE +POSTHOOK: query: drop database if exists statsdb1 +POSTHOOK: type: DROPDATABASE +PREHOOK: query: create database statsdb1 +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:statsdb1 +POSTHOOK: query: create database statsdb1 +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:statsdb1 +PREHOOK: query: drop database if exists statsdb2 +PREHOOK: type: DROPDATABASE +POSTHOOK: query: drop database if exists statsdb2 +POSTHOOK: type: DROPDATABASE +PREHOOK: query: create database statsdb2 +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:statsdb2 +POSTHOOK: query: create database statsdb2 +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:statsdb2 +PREHOOK: query: create table statsdb1.testtable1 (col1 int, col2 string, col3 string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:statsdb1 +PREHOOK: Output: statsdb1@testtable1 +POSTHOOK: query: create table statsdb1.testtable1 (col1 int, col2 string, col3 string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:statsdb1 +POSTHOOK: Output: statsdb1@testtable1 +PREHOOK: query: insert into statsdb1.testtable1 select key, value, 'val3' from src limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: statsdb1@testtable1 +POSTHOOK: query: insert into statsdb1.testtable1 select key, value, 'val3' from src limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: statsdb1@testtable1 +POSTHOOK: Lineage: testtable1.col1 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: testtable1.col2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: testtable1.col3 SIMPLE [] +PREHOOK: query: create table statsdb1.testpart1 (col1 int, col2 string, col3 string) partitioned by (part string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:statsdb1 +PREHOOK: Output: statsdb1@testpart1 +POSTHOOK: query: create table statsdb1.testpart1 (col1 int, col2 string, col3 string) partitioned by (part string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:statsdb1 +POSTHOOK: Output: statsdb1@testpart1 +PREHOOK: query: insert into statsdb1.testpart1 partition (part = 'part1') select key, value, 'val3' from src limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: statsdb1@testpart1@part=part1 +POSTHOOK: query: insert into statsdb1.testpart1 partition (part = 'part1') select key, value, 'val3' from src limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: statsdb1@testpart1@part=part1 +POSTHOOK: Lineage: testpart1 PARTITION(part=part1).col1 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: testpart1 PARTITION(part=part1).col2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: testpart1 PARTITION(part=part1).col3 SIMPLE [] +PREHOOK: query: insert into statsdb1.testpart1 partition (part = 'part2') select key, value, 'val3' from src limit 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: statsdb1@testpart1@part=part2 +POSTHOOK: query: insert into statsdb1.testpart1 partition (part = 'part2') select key, value, 'val3' from src limit 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: statsdb1@testpart1@part=part2 +POSTHOOK: Lineage: testpart1 PARTITION(part=part2).col1 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: testpart1 PARTITION(part=part2).col2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: testpart1 PARTITION(part=part2).col3 SIMPLE [] +PREHOOK: query: use statsdb1 +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:statsdb1 +POSTHOOK: query: use statsdb1 +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:statsdb1 +PREHOOK: query: analyze table testtable1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: statsdb1@testtable1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table testtable1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: statsdb1@testtable1 +#### A masked pattern was here #### +PREHOOK: query: describe formatted statsdb1.testtable1 col1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testtable1 +POSTHOOK: query: describe formatted statsdb1.testtable1 col1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testtable1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col1 int 27 484 0 8 from deserializer +PREHOOK: query: describe formatted statsdb1.testtable1 col2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testtable1 +POSTHOOK: query: describe formatted statsdb1.testtable1 col2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testtable1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col2 string 0 12 6.7 7 from deserializer +PREHOOK: query: describe formatted statsdb1.testtable1 col3 +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testtable1 +POSTHOOK: query: describe formatted statsdb1.testtable1 col3 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testtable1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col3 string 0 1 4.0 4 from deserializer +PREHOOK: query: alter table testtable1 replace columns (col1 int, col2 string, col4 string) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: statsdb1@testtable1 +PREHOOK: Output: statsdb1@testtable1 +POSTHOOK: query: alter table testtable1 replace columns (col1 int, col2 string, col4 string) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: statsdb1@testtable1 +POSTHOOK: Output: statsdb1@testtable1 +PREHOOK: query: describe formatted statsdb1.testtable1 col1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testtable1 +POSTHOOK: query: describe formatted statsdb1.testtable1 col1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testtable1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col1 int 27 484 0 8 from deserializer +PREHOOK: query: describe formatted statsdb1.testtable1 col2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testtable1 +POSTHOOK: query: describe formatted statsdb1.testtable1 col2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testtable1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col2 string 0 12 6.7 7 from deserializer +PREHOOK: query: describe formatted statsdb1.testtable1 col4 +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testtable1 +POSTHOOK: query: describe formatted statsdb1.testtable1 col4 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testtable1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col4 string from deserializer +PREHOOK: query: alter table testtable1 change col1 col1 string +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: statsdb1@testtable1 +PREHOOK: Output: statsdb1@testtable1 +POSTHOOK: query: alter table testtable1 change col1 col1 string +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: statsdb1@testtable1 +POSTHOOK: Output: statsdb1@testtable1 +PREHOOK: query: describe formatted statsdb1.testtable1 col1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testtable1 +POSTHOOK: query: describe formatted statsdb1.testtable1 col1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testtable1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col1 string from deserializer +PREHOOK: query: describe formatted statsdb1.testtable1 col2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testtable1 +POSTHOOK: query: describe formatted statsdb1.testtable1 col2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testtable1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col2 string 0 12 6.7 7 from deserializer +PREHOOK: query: describe formatted statsdb1.testtable1 col4 +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testtable1 +POSTHOOK: query: describe formatted statsdb1.testtable1 col4 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testtable1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col4 string from deserializer +PREHOOK: query: alter table statsdb1.testtable1 rename to statsdb2.testtable2 +PREHOOK: type: ALTERTABLE_RENAME +PREHOOK: Input: statsdb1@testtable1 +PREHOOK: Output: statsdb1@testtable1 +POSTHOOK: query: alter table statsdb1.testtable1 rename to statsdb2.testtable2 +POSTHOOK: type: ALTERTABLE_RENAME +POSTHOOK: Input: statsdb1@testtable1 +POSTHOOK: Output: statsdb1@testtable1 +POSTHOOK: Output: statsdb2@testtable2 +PREHOOK: query: analyze table testpart1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: statsdb1@testpart1 +PREHOOK: Input: statsdb1@testpart1@part=part1 +PREHOOK: Input: statsdb1@testpart1@part=part2 +#### A masked pattern was here #### +POSTHOOK: query: analyze table testpart1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: statsdb1@testpart1 +POSTHOOK: Input: statsdb1@testpart1@part=part1 +POSTHOOK: Input: statsdb1@testpart1@part=part2 +#### A masked pattern was here #### +PREHOOK: query: describe formatted statsdb1.testpart1 col1 partition (part = 'part1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col1 partition (part = 'part1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col1 int 27 484 0 8 from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col2 partition (part = 'part1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col2 partition (part = 'part1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col2 string 0 12 6.7 7 from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col3 partition (part = 'part1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col3 partition (part = 'part1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col3 string 0 1 4.0 4 from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col1 partition (part = 'part2') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col1 partition (part = 'part2') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col1 int 27 484 0 18 from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col2 partition (part = 'part2') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col2 partition (part = 'part2') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col2 string 0 18 6.8 7 from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col3 partition (part = 'part2') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col3 partition (part = 'part2') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col3 string 0 1 4.0 4 from deserializer +PREHOOK: query: alter table statsdb1.testpart1 partition (part = 'part2') rename to partition (part = 'part3') +PREHOOK: type: ALTERTABLE_RENAMEPART +PREHOOK: Input: statsdb1@testpart1 +PREHOOK: Output: statsdb1@testpart1@part=part2 +POSTHOOK: query: alter table statsdb1.testpart1 partition (part = 'part2') rename to partition (part = 'part3') +POSTHOOK: type: ALTERTABLE_RENAMEPART +POSTHOOK: Input: statsdb1@testpart1 +POSTHOOK: Input: statsdb1@testpart1@part=part2 +POSTHOOK: Output: statsdb1@testpart1@part=part2 +POSTHOOK: Output: statsdb1@testpart1@part=part3 +PREHOOK: query: describe formatted statsdb1.testpart1 col1 partition (part = 'part1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col1 partition (part = 'part1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col1 int 27 484 0 8 from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col2 partition (part = 'part1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col2 partition (part = 'part1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col2 string 0 12 6.7 7 from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col3 partition (part = 'part1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col3 partition (part = 'part1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col3 string 0 1 4.0 4 from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col1 partition (part = 'part3') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col1 partition (part = 'part3') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type comment + +col1 int from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col2 partition (part = 'part3') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col2 partition (part = 'part3') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type comment + +col2 string from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col3 partition (part = 'part3') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col3 partition (part = 'part3') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type comment + +col3 string from deserializer +PREHOOK: query: alter table statsdb1.testpart1 replace columns (col1 int, col2 string, col4 string) cascade +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: statsdb1@testpart1 +PREHOOK: Output: statsdb1@testpart1 +PREHOOK: Output: statsdb1@testpart1@part=part1 +PREHOOK: Output: statsdb1@testpart1@part=part3 +POSTHOOK: query: alter table statsdb1.testpart1 replace columns (col1 int, col2 string, col4 string) cascade +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: statsdb1@testpart1 +POSTHOOK: Output: statsdb1@testpart1 +POSTHOOK: Output: statsdb1@testpart1@part=part1 +POSTHOOK: Output: statsdb1@testpart1@part=part3 +PREHOOK: query: describe formatted statsdb1.testpart1 col1 partition (part = 'part1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col1 partition (part = 'part1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col1 int 27 484 0 8 from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col2 partition (part = 'part1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col2 partition (part = 'part1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col2 string 0 12 6.7 7 from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col4 partition (part = 'part1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col4 partition (part = 'part1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type comment + +col4 string from deserializer +PREHOOK: query: alter table statsdb1.testpart1 change column col1 col1 string +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: statsdb1@testpart1 +PREHOOK: Output: statsdb1@testpart1 +POSTHOOK: query: alter table statsdb1.testpart1 change column col1 col1 string +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: statsdb1@testpart1 +POSTHOOK: Output: statsdb1@testpart1 +PREHOOK: query: alter table statsdb1.testpart1 partition (part) change column col1 col1 string +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: statsdb1@testpart1 +PREHOOK: Output: statsdb1@testpart1@part=part1 +PREHOOK: Output: statsdb1@testpart1@part=part3 +POSTHOOK: query: alter table statsdb1.testpart1 partition (part) change column col1 col1 string +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: statsdb1@testpart1 +POSTHOOK: Input: statsdb1@testpart1@part=part1 +POSTHOOK: Input: statsdb1@testpart1@part=part3 +POSTHOOK: Output: statsdb1@testpart1@part=part1 +POSTHOOK: Output: statsdb1@testpart1@part=part3 +PREHOOK: query: describe formatted statsdb1.testpart1 col1 partition (part = 'part1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col1 partition (part = 'part1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type comment + +col1 string from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col2 partition (part = 'part1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col2 partition (part = 'part1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col2 string 0 12 6.7 7 from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col4 partition (part = 'part1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col4 partition (part = 'part1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type comment + +col4 string from deserializer +PREHOOK: query: alter table statsdb1.testpart1 rename to statsdb2.testpart2 +PREHOOK: type: ALTERTABLE_RENAME +PREHOOK: Input: statsdb1@testpart1 +PREHOOK: Output: statsdb1@testpart1 +POSTHOOK: query: alter table statsdb1.testpart1 rename to statsdb2.testpart2 +POSTHOOK: type: ALTERTABLE_RENAME +POSTHOOK: Input: statsdb1@testpart1 +POSTHOOK: Output: statsdb1@testpart1 +POSTHOOK: Output: statsdb2@testpart2 +PREHOOK: query: use statsdb2 +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:statsdb2 +POSTHOOK: query: use statsdb2 +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:statsdb2 +PREHOOK: query: alter table statsdb2.testpart2 drop partition (part = 'part1') +PREHOOK: type: ALTERTABLE_DROPPARTS +PREHOOK: Input: statsdb2@testpart2 +PREHOOK: Output: statsdb2@testpart2@part=part1 +POSTHOOK: query: alter table statsdb2.testpart2 drop partition (part = 'part1') +POSTHOOK: type: ALTERTABLE_DROPPARTS +POSTHOOK: Input: statsdb2@testpart2 +POSTHOOK: Output: statsdb2@testpart2@part=part1 +PREHOOK: query: drop table statsdb2.testpart2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: statsdb2@testpart2 +PREHOOK: Output: statsdb2@testpart2 +POSTHOOK: query: drop table statsdb2.testpart2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: statsdb2@testpart2 +POSTHOOK: Output: statsdb2@testpart2 +PREHOOK: query: drop table statsdb2.testtable2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: statsdb2@testtable2 +PREHOOK: Output: statsdb2@testtable2 +POSTHOOK: query: drop table statsdb2.testtable2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: statsdb2@testtable2 +POSTHOOK: Output: statsdb2@testtable2 +PREHOOK: query: use default +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:default +POSTHOOK: query: use default +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:default +PREHOOK: query: drop database statsdb1 +PREHOOK: type: DROPDATABASE +PREHOOK: Input: database:statsdb1 +PREHOOK: Output: database:statsdb1 +POSTHOOK: query: drop database statsdb1 +POSTHOOK: type: DROPDATABASE +POSTHOOK: Input: database:statsdb1 +POSTHOOK: Output: database:statsdb1 +PREHOOK: query: drop database statsdb2 +PREHOOK: type: DROPDATABASE +PREHOOK: Input: database:statsdb2 +PREHOOK: Output: database:statsdb2 +POSTHOOK: query: drop database statsdb2 +POSTHOOK: type: DROPDATABASE +POSTHOOK: Input: database:statsdb2 +POSTHOOK: Output: database:statsdb2 +PREHOOK: query: drop database if exists statsdb1 +PREHOOK: type: DROPDATABASE +POSTHOOK: query: drop database if exists statsdb1 +POSTHOOK: type: DROPDATABASE +PREHOOK: query: create database statsdb1 +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:statsdb1 +POSTHOOK: query: create database statsdb1 +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:statsdb1 +PREHOOK: query: drop database if exists statsdb2 +PREHOOK: type: DROPDATABASE +POSTHOOK: query: drop database if exists statsdb2 +POSTHOOK: type: DROPDATABASE +PREHOOK: query: create database statsdb2 +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:statsdb2 +POSTHOOK: query: create database statsdb2 +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:statsdb2 +PREHOOK: query: create table statsdb1.testtable1 (col1 int, col2 string, col3 string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:statsdb1 +PREHOOK: Output: statsdb1@testtable1 +POSTHOOK: query: create table statsdb1.testtable1 (col1 int, col2 string, col3 string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:statsdb1 +POSTHOOK: Output: statsdb1@testtable1 +PREHOOK: query: insert into statsdb1.testtable1 select key, value, 'val3' from src limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: statsdb1@testtable1 +POSTHOOK: query: insert into statsdb1.testtable1 select key, value, 'val3' from src limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: statsdb1@testtable1 +POSTHOOK: Lineage: testtable1.col1 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: testtable1.col2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: testtable1.col3 SIMPLE [] +PREHOOK: query: create table statsdb1.testpart1 (col1 int, col2 string, col3 string) partitioned by (part string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:statsdb1 +PREHOOK: Output: statsdb1@testpart1 +POSTHOOK: query: create table statsdb1.testpart1 (col1 int, col2 string, col3 string) partitioned by (part string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:statsdb1 +POSTHOOK: Output: statsdb1@testpart1 +PREHOOK: query: insert into statsdb1.testpart1 partition (part = 'part1') select key, value, 'val3' from src limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: statsdb1@testpart1@part=part1 +POSTHOOK: query: insert into statsdb1.testpart1 partition (part = 'part1') select key, value, 'val3' from src limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: statsdb1@testpart1@part=part1 +POSTHOOK: Lineage: testpart1 PARTITION(part=part1).col1 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: testpart1 PARTITION(part=part1).col2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: testpart1 PARTITION(part=part1).col3 SIMPLE [] +PREHOOK: query: insert into statsdb1.testpart1 partition (part = 'part2') select key, value, 'val3' from src limit 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: statsdb1@testpart1@part=part2 +POSTHOOK: query: insert into statsdb1.testpart1 partition (part = 'part2') select key, value, 'val3' from src limit 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: statsdb1@testpart1@part=part2 +POSTHOOK: Lineage: testpart1 PARTITION(part=part2).col1 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: testpart1 PARTITION(part=part2).col2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: testpart1 PARTITION(part=part2).col3 SIMPLE [] +PREHOOK: query: use statsdb1 +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:statsdb1 +POSTHOOK: query: use statsdb1 +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:statsdb1 +PREHOOK: query: analyze table testtable1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: statsdb1@testtable1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table testtable1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: statsdb1@testtable1 +#### A masked pattern was here #### +PREHOOK: query: describe formatted statsdb1.testtable1 col1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testtable1 +POSTHOOK: query: describe formatted statsdb1.testtable1 col1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testtable1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col1 int 27 484 0 8 from deserializer +PREHOOK: query: describe formatted statsdb1.testtable1 col2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testtable1 +POSTHOOK: query: describe formatted statsdb1.testtable1 col2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testtable1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col2 string 0 12 6.7 7 from deserializer +PREHOOK: query: describe formatted statsdb1.testtable1 col3 +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testtable1 +POSTHOOK: query: describe formatted statsdb1.testtable1 col3 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testtable1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col3 string 0 1 4.0 4 from deserializer +PREHOOK: query: alter table testtable1 replace columns (col1 int, col2 string, col4 string) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: statsdb1@testtable1 +PREHOOK: Output: statsdb1@testtable1 +POSTHOOK: query: alter table testtable1 replace columns (col1 int, col2 string, col4 string) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: statsdb1@testtable1 +POSTHOOK: Output: statsdb1@testtable1 +PREHOOK: query: describe formatted statsdb1.testtable1 col1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testtable1 +POSTHOOK: query: describe formatted statsdb1.testtable1 col1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testtable1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col1 int 27 484 0 8 from deserializer +PREHOOK: query: describe formatted statsdb1.testtable1 col2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testtable1 +POSTHOOK: query: describe formatted statsdb1.testtable1 col2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testtable1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col2 string 0 12 6.7 7 from deserializer +PREHOOK: query: describe formatted statsdb1.testtable1 col4 +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testtable1 +POSTHOOK: query: describe formatted statsdb1.testtable1 col4 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testtable1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col4 string from deserializer +PREHOOK: query: alter table testtable1 change col1 col1 string +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: statsdb1@testtable1 +PREHOOK: Output: statsdb1@testtable1 +POSTHOOK: query: alter table testtable1 change col1 col1 string +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: statsdb1@testtable1 +POSTHOOK: Output: statsdb1@testtable1 +PREHOOK: query: describe formatted statsdb1.testtable1 col1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testtable1 +POSTHOOK: query: describe formatted statsdb1.testtable1 col1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testtable1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col1 string from deserializer +PREHOOK: query: describe formatted statsdb1.testtable1 col2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testtable1 +POSTHOOK: query: describe formatted statsdb1.testtable1 col2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testtable1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col2 string 0 12 6.7 7 from deserializer +PREHOOK: query: describe formatted statsdb1.testtable1 col4 +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testtable1 +POSTHOOK: query: describe formatted statsdb1.testtable1 col4 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testtable1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col4 string from deserializer +PREHOOK: query: alter table statsdb1.testtable1 rename to statsdb2.testtable2 +PREHOOK: type: ALTERTABLE_RENAME +PREHOOK: Input: statsdb1@testtable1 +PREHOOK: Output: statsdb1@testtable1 +POSTHOOK: query: alter table statsdb1.testtable1 rename to statsdb2.testtable2 +POSTHOOK: type: ALTERTABLE_RENAME +POSTHOOK: Input: statsdb1@testtable1 +POSTHOOK: Output: statsdb1@testtable1 +POSTHOOK: Output: statsdb2@testtable2 +PREHOOK: query: analyze table testpart1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: statsdb1@testpart1 +PREHOOK: Input: statsdb1@testpart1@part=part1 +PREHOOK: Input: statsdb1@testpart1@part=part2 +#### A masked pattern was here #### +POSTHOOK: query: analyze table testpart1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: statsdb1@testpart1 +POSTHOOK: Input: statsdb1@testpart1@part=part1 +POSTHOOK: Input: statsdb1@testpart1@part=part2 +#### A masked pattern was here #### +PREHOOK: query: describe formatted statsdb1.testpart1 col1 partition (part = 'part1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col1 partition (part = 'part1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col1 int 27 484 0 8 from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col2 partition (part = 'part1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col2 partition (part = 'part1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col2 string 0 12 6.7 7 from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col3 partition (part = 'part1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col3 partition (part = 'part1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col3 string 0 1 4.0 4 from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col1 partition (part = 'part2') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col1 partition (part = 'part2') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col1 int 27 484 0 18 from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col2 partition (part = 'part2') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col2 partition (part = 'part2') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col2 string 0 18 6.8 7 from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col3 partition (part = 'part2') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col3 partition (part = 'part2') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col3 string 0 1 4.0 4 from deserializer +PREHOOK: query: alter table statsdb1.testpart1 partition (part = 'part2') rename to partition (part = 'part3') +PREHOOK: type: ALTERTABLE_RENAMEPART +PREHOOK: Input: statsdb1@testpart1 +PREHOOK: Output: statsdb1@testpart1@part=part2 +POSTHOOK: query: alter table statsdb1.testpart1 partition (part = 'part2') rename to partition (part = 'part3') +POSTHOOK: type: ALTERTABLE_RENAMEPART +POSTHOOK: Input: statsdb1@testpart1 +POSTHOOK: Input: statsdb1@testpart1@part=part2 +POSTHOOK: Output: statsdb1@testpart1@part=part2 +POSTHOOK: Output: statsdb1@testpart1@part=part3 +PREHOOK: query: describe formatted statsdb1.testpart1 col1 partition (part = 'part1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col1 partition (part = 'part1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col1 int 27 484 0 8 from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col2 partition (part = 'part1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col2 partition (part = 'part1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col2 string 0 12 6.7 7 from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col3 partition (part = 'part1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col3 partition (part = 'part1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col3 string 0 1 4.0 4 from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col1 partition (part = 'part3') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col1 partition (part = 'part3') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type comment + +col1 int from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col2 partition (part = 'part3') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col2 partition (part = 'part3') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type comment + +col2 string from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col3 partition (part = 'part3') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col3 partition (part = 'part3') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type comment + +col3 string from deserializer +PREHOOK: query: alter table statsdb1.testpart1 replace columns (col1 int, col2 string, col4 string) cascade +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: statsdb1@testpart1 +PREHOOK: Output: statsdb1@testpart1 +PREHOOK: Output: statsdb1@testpart1@part=part1 +PREHOOK: Output: statsdb1@testpart1@part=part3 +POSTHOOK: query: alter table statsdb1.testpart1 replace columns (col1 int, col2 string, col4 string) cascade +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: statsdb1@testpart1 +POSTHOOK: Output: statsdb1@testpart1 +POSTHOOK: Output: statsdb1@testpart1@part=part1 +POSTHOOK: Output: statsdb1@testpart1@part=part3 +PREHOOK: query: describe formatted statsdb1.testpart1 col1 partition (part = 'part1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col1 partition (part = 'part1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col1 int 27 484 0 8 from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col2 partition (part = 'part1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col2 partition (part = 'part1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col2 string 0 12 6.7 7 from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col4 partition (part = 'part1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col4 partition (part = 'part1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type comment + +col4 string from deserializer +PREHOOK: query: alter table statsdb1.testpart1 change column col1 col1 string +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: statsdb1@testpart1 +PREHOOK: Output: statsdb1@testpart1 +POSTHOOK: query: alter table statsdb1.testpart1 change column col1 col1 string +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: statsdb1@testpart1 +POSTHOOK: Output: statsdb1@testpart1 +PREHOOK: query: alter table statsdb1.testpart1 partition (part) change column col1 col1 string +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: statsdb1@testpart1 +PREHOOK: Output: statsdb1@testpart1@part=part1 +PREHOOK: Output: statsdb1@testpart1@part=part3 +POSTHOOK: query: alter table statsdb1.testpart1 partition (part) change column col1 col1 string +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: statsdb1@testpart1 +POSTHOOK: Input: statsdb1@testpart1@part=part1 +POSTHOOK: Input: statsdb1@testpart1@part=part3 +POSTHOOK: Output: statsdb1@testpart1@part=part1 +POSTHOOK: Output: statsdb1@testpart1@part=part3 +PREHOOK: query: describe formatted statsdb1.testpart1 col1 partition (part = 'part1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col1 partition (part = 'part1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type comment + +col1 string from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col2 partition (part = 'part1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col2 partition (part = 'part1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +col2 string 0 12 6.7 7 from deserializer +PREHOOK: query: describe formatted statsdb1.testpart1 col4 partition (part = 'part1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: statsdb1@testpart1 +POSTHOOK: query: describe formatted statsdb1.testpart1 col4 partition (part = 'part1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: statsdb1@testpart1 +# col_name data_type comment + +col4 string from deserializer +PREHOOK: query: alter table statsdb1.testpart1 rename to statsdb2.testpart2 +PREHOOK: type: ALTERTABLE_RENAME +PREHOOK: Input: statsdb1@testpart1 +PREHOOK: Output: statsdb1@testpart1 +POSTHOOK: query: alter table statsdb1.testpart1 rename to statsdb2.testpart2 +POSTHOOK: type: ALTERTABLE_RENAME +POSTHOOK: Input: statsdb1@testpart1 +POSTHOOK: Output: statsdb1@testpart1 +POSTHOOK: Output: statsdb2@testpart2 +PREHOOK: query: use statsdb2 +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:statsdb2 +POSTHOOK: query: use statsdb2 +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:statsdb2 +PREHOOK: query: alter table statsdb2.testpart2 drop partition (part = 'part1') +PREHOOK: type: ALTERTABLE_DROPPARTS +PREHOOK: Input: statsdb2@testpart2 +PREHOOK: Output: statsdb2@testpart2@part=part1 +POSTHOOK: query: alter table statsdb2.testpart2 drop partition (part = 'part1') +POSTHOOK: type: ALTERTABLE_DROPPARTS +POSTHOOK: Input: statsdb2@testpart2 +POSTHOOK: Output: statsdb2@testpart2@part=part1 +PREHOOK: query: drop table statsdb2.testpart2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: statsdb2@testpart2 +PREHOOK: Output: statsdb2@testpart2 +POSTHOOK: query: drop table statsdb2.testpart2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: statsdb2@testpart2 +POSTHOOK: Output: statsdb2@testpart2 +PREHOOK: query: drop table statsdb2.testtable2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: statsdb2@testtable2 +PREHOOK: Output: statsdb2@testtable2 +POSTHOOK: query: drop table statsdb2.testtable2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: statsdb2@testtable2 +POSTHOOK: Output: statsdb2@testtable2 +PREHOOK: query: use default +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:default +POSTHOOK: query: use default +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:default +PREHOOK: query: drop database statsdb1 +PREHOOK: type: DROPDATABASE +PREHOOK: Input: database:statsdb1 +PREHOOK: Output: database:statsdb1 +POSTHOOK: query: drop database statsdb1 +POSTHOOK: type: DROPDATABASE +POSTHOOK: Input: database:statsdb1 +POSTHOOK: Output: database:statsdb1 +PREHOOK: query: drop database statsdb2 +PREHOOK: type: DROPDATABASE +PREHOOK: Input: database:statsdb2 +PREHOOK: Output: database:statsdb2 +POSTHOOK: query: drop database statsdb2 +POSTHOOK: type: DROPDATABASE +POSTHOOK: Input: database:statsdb2 +POSTHOOK: Output: database:statsdb2