diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java index 76c1636..3803971 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java @@ -103,7 +103,7 @@ protected void setUp() { db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, src, true, true); db.createTable(src, cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class); - db.loadTable(hadoopDataFile[i], src, false, false, false, false, false); + db.loadTable(hadoopDataFile[i], src, false, false, false, false, false, false); i++; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index 38dc406..1e3baa0 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -324,6 +324,16 @@ public static boolean updatePartitionStatsFast(Partition part, Warehouse wh, public static boolean updatePartitionStatsFast(PartitionSpecProxy.PartitionIterator part, Warehouse wh, boolean madeDir, boolean forceRecompute) throws MetaException { Map params = part.getParameters(); + + if ((params!=null) && params.containsKey(StatsSetupConst.DO_NOT_UPDATE_STATS)){ + boolean doNotUpdateStats = Boolean.valueOf(params.get(StatsSetupConst.DO_NOT_UPDATE_STATS)); + params.remove(StatsSetupConst.DO_NOT_UPDATE_STATS); + part.setParameters(params); // to make sure we remove this marker property + if (doNotUpdateStats){ + return false; + } + } + boolean updated = false; if (forceRecompute || params == null || diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java index b07a37a..c80bb07 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java @@ -297,7 +297,8 @@ public int execute(DriverContext driverContext) { db.loadTable(tbd.getSourcePath(), tbd.getTable() .getTableName(), tbd.getReplace(), tbd.getHoldDDLTime(), work.isSrcLocal(), isSkewedStoredAsDirs(tbd), - work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID); + work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID, + work.isInImportScope()); if (work.getOutputs() != null) { work.getOutputs().add(new WriteEntity(table, (tbd.getReplace() ? WriteEntity.WriteType.INSERT_OVERWRITE : diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index bc99b22..3d26278 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -1630,11 +1630,19 @@ private void constructOneLBLocationMap(FileStatus fSta, * @param isAcid true if this is an ACID based write */ public void loadTable(Path loadPath, String tableName, boolean replace, - boolean holdDDLTime, boolean isSrcLocal, boolean isSkewedStoreAsSubdir, boolean isAcid) + boolean holdDDLTime, boolean isSrcLocal, boolean isSkewedStoreAsSubdir, + boolean isAcid, boolean isInImportScope) throws HiveException { List newFiles = new ArrayList(); Table tbl = getTable(tableName); HiveConf sessionConf = SessionState.getSessionConf(); + + if (isInImportScope){ + // If we're importing, then we want to retain the metadata that was recorded by the export + // rather than resetting it. + tbl.getParameters().put(StatsSetupConst.DO_NOT_UPDATE_STATS, "true"); + } + if (replace) { Path tableDest = tbl.getPath(); replaceFiles(tableDest, loadPath, tableDest, tableDest, sessionConf, isSrcLocal); @@ -1663,7 +1671,7 @@ public void loadTable(Path loadPath, String tableName, boolean replace, throw new HiveException(e); } - if (!holdDDLTime) { + if (!holdDDLTime && !isInImportScope) { try { alterTable(tableName, tbl); } catch (InvalidOperationException e) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java index 1061cb0..82c65d9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java @@ -266,6 +266,9 @@ private void parsePartitionSpec(ASTNode tableNode, LinkedHashMap private AddPartitionDesc getBaseAddPartitionDescFromPartition( Path fromPath, String dbname, CreateTableDesc tblDesc, Partition partition) throws MetaException { + if (partition.getParameters() != null){ + partition.putToParameters(StatsSetupConst.DO_NOT_UPDATE_STATS,"true"); + } AddPartitionDesc partsDesc = new AddPartitionDesc(dbname, tblDesc.getTableName(), EximUtil.makePartSpec(tblDesc.getPartCols(), partition.getValues()), partition.getSd().getLocation(), partition.getParameters()); @@ -325,8 +328,9 @@ private CreateTableDesc getBaseCreateTableDescFromTable(String dbName, LoadTableDesc loadTableWork = new LoadTableDesc(tmpPath, Utilities.getTableDesc(table), new TreeMap(), replace); - Task loadTableTask = TaskFactory.get(new MoveWork(getInputs(), - getOutputs(), loadTableWork, null, false), conf); + MoveWork moveWork = new MoveWork(getInputs(), getOutputs(), loadTableWork, null, false); + moveWork.setInImportScope(true); + Task loadTableTask = TaskFactory.get( moveWork, conf); copyTask.addDependentTask(loadTableTask); rootTasks.add(copyTask); return loadTableTask; @@ -363,6 +367,7 @@ private CreateTableDesc getBaseCreateTableDescFromTable(String dbName, ReplicationSpec replicationSpec, org.apache.hadoop.hive.ql.metadata.Partition ptn) { addPartitionDesc.setReplaceMode(true); addPartitionDesc.getPartition(0).setLocation(ptn.getLocation()); // use existing location + addPartitionDesc.getPartition(0).getPartParams().put(StatsSetupConst.DO_NOT_UPDATE_STATS,"true"); return TaskFactory.get(new DDLWork( getInputs(), getOutputs(), @@ -398,9 +403,9 @@ private CreateTableDesc getBaseCreateTableDescFromTable(String dbName, Utilities.getTableDesc(table), partSpec.getPartSpec(), true); loadTableWork.setInheritTableSpecs(false); - Task loadPartTask = TaskFactory.get(new MoveWork( - getInputs(), getOutputs(), loadTableWork, null, false), - conf); + MoveWork moveWork = new MoveWork(getInputs(), getOutputs(), loadTableWork, null, false); + moveWork.setInImportScope(true); + Task loadPartTask = TaskFactory.get(moveWork, conf); copyTask.addDependentTask(loadPartTask); addPartTask.addDependentTask(loadPartTask); rootTasks.add(copyTask); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java index 9f498c7..8652386 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java @@ -41,6 +41,7 @@ private boolean checkFileFormat; private boolean srcLocal; + private boolean isInImportScope = false; /** * ReadEntitites that are passed to the hooks. @@ -142,4 +143,12 @@ public void setSrcLocal(boolean srcLocal) { this.srcLocal = srcLocal; } + public boolean isInImportScope() { + return isInImportScope; + } + + public void setInImportScope(boolean isInImportScope) { + this.isInImportScope = isInImportScope; + } + } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java index ca59e90..7ac29e9 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java @@ -139,7 +139,7 @@ db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, src, true, true); db.createTable(src, cols, null, TextInputFormat.class, HiveIgnoreKeyTextOutputFormat.class); - db.loadTable(hadoopDataFile[i], src, false, false, true, false, false); + db.loadTable(hadoopDataFile[i], src, false, false, true, false, false, false); i++; } diff --git a/ql/src/test/queries/clientpositive/repl_4_stats_retain.q b/ql/src/test/queries/clientpositive/repl_4_stats_retain.q new file mode 100644 index 0000000..c8cd055 --- /dev/null +++ b/ql/src/test/queries/clientpositive/repl_4_stats_retain.q @@ -0,0 +1,103 @@ +set hive.test.mode=true; +set hive.test.mode.prefix=; +set hive.test.mode.nosamplelist=ptn_t,import_ptn_t_1,import_ptn_t_2,unptn_t,import_unptn_t_1,import_unptn_t_2; + +drop table if exists ptn_t; +drop table if exists unptn_t; +drop table if exists import_ptn_t_1; +drop table if exists import_ptn_t_2; +drop table if exists import_unptn_t_1; +drop table if exists import_unptn_t_2; + +dfs ${system:test.dfs.mkdir} target/tmp/ql/test/data/exports/ptn_t/temp; +dfs -rmr target/tmp/ql/test/data/exports/ptn_t; +dfs ${system:test.dfs.mkdir} target/tmp/ql/test/data/exports/unptn_t/temp; +dfs -rmr target/tmp/ql/test/data/exports/unptn_t; + +dfs ${system:test.dfs.mkdir} target/tmp/ql/test/data/exports/ptn_t_1/temp; +dfs -rmr target/tmp/ql/test/data/exports/ptn_t_1; +dfs ${system:test.dfs.mkdir} target/tmp/ql/test/data/exports/ptn_t_2/temp; +dfs -rmr target/tmp/ql/test/data/exports/ptn_t_2; + +dfs ${system:test.dfs.mkdir} target/tmp/ql/test/data/exports/unptn_t_1/temp; +dfs -rmr target/tmp/ql/test/data/exports/unptn_t_1; +dfs ${system:test.dfs.mkdir} target/tmp/ql/test/data/exports/unptn_t_2/temp; +dfs -rmr target/tmp/ql/test/data/exports/unptn_t_2; + + +create table ptn_t (emp_id int comment "employee id") + partitioned by (emp_country string, emp_state string) + stored as textfile; +load data local inpath "../../data/files/test.dat" + into table ptn_t partition (emp_country="us",emp_state="ca"); + +create table unptn_t(emp_id int comment "employee id") + stored as textfile; +load data local inpath "../../data/files/test.dat" + into table unptn_t; + +-- We create a table, and export the table to an export location before and after an analyze table compute statistics + +describe extended ptn_t; +show table extended like 'ptn_t' partition(emp_country="us",emp_state="ca"); +show create table ptn_t; + +export table ptn_t to 'ql/test/data/exports/ptn_t_1' for replication('ptn_t_1'); + +analyze table ptn_t partition(emp_country="us",emp_state="ca") compute statistics; + +describe extended ptn_t; +show table extended like 'ptn_t' partition(emp_country="us",emp_state="ca"); +show create table ptn_t; + +export table ptn_t to 'ql/test/data/exports/ptn_t_2' for replication('ptn_t_2'); + +drop table ptn_t; + +-- Now the same for the unpartitioned table + +describe extended unptn_t; +show create table unptn_t; + +export table unptn_t to 'ql/test/data/exports/unptn_t_1' for replication('unptn_t_1'); + +analyze table unptn_t compute statistics; + +describe extended unptn_t; +show create table unptn_t; + +export table unptn_t to 'ql/test/data/exports/unptn_t_2' for replication('unptn_t_2'); + +drop table unptn_t; + +-- Now we import + +explain import table import_ptn_t_1 from 'ql/test/data/exports/ptn_t_1'; +import table import_ptn_t_1 from 'ql/test/data/exports/ptn_t_1'; +describe extended import_ptn_t_1; +show table extended like 'import_ptn_t_1' partition(emp_country="us",emp_state="ca"); +show create table import_ptn_t_1; + +explain import table import_ptn_t_2 from 'ql/test/data/exports/ptn_t_2'; +import table import_ptn_t_2 from 'ql/test/data/exports/ptn_t_2'; +describe extended import_ptn_t_2; +show table extended like 'import_ptn_t_2' partition(emp_country="us",emp_state="ca"); +show create table import_ptn_t_2; + +import table import_unptn_t_1 from 'ql/test/data/exports/unptn_t_1'; +describe extended import_unptn_t_1; +show table extended like 'import_unptn_t_1'; +show create table import_unptn_t_1; + +import table import_unptn_t_2 from 'ql/test/data/exports/unptn_t_2'; +describe extended import_unptn_t_2; +show table extended like 'import_unptn_t_2'; +show create table import_unptn_t_2; + +-- clean up + +drop table import_ptn_t_1; +drop table import_ptn_t_2; +drop table import_unptn_t_1; +drop table import_unptn_t_2; + diff --git a/ql/src/test/results/clientpositive/repl_4_stats_retain.q.out b/ql/src/test/results/clientpositive/repl_4_stats_retain.q.out new file mode 100644 index 0000000..35f01c7 --- /dev/null +++ b/ql/src/test/results/clientpositive/repl_4_stats_retain.q.out @@ -0,0 +1,698 @@ +PREHOOK: query: drop table if exists ptn_t +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists ptn_t +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table if exists unptn_t +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists unptn_t +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table if exists import_ptn_t_1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists import_ptn_t_1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table if exists import_ptn_t_2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists import_ptn_t_2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table if exists import_unptn_t_1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists import_unptn_t_1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table if exists import_unptn_t_2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists import_unptn_t_2 +POSTHOOK: type: DROPTABLE +#### A masked pattern was here #### +PREHOOK: query: create table ptn_t (emp_id int comment "employee id") + partitioned by (emp_country string, emp_state string) + stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ptn_t +POSTHOOK: query: create table ptn_t (emp_id int comment "employee id") + partitioned by (emp_country string, emp_state string) + stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ptn_t +PREHOOK: query: load data local inpath "../../data/files/test.dat" + into table ptn_t partition (emp_country="us",emp_state="ca") +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@ptn_t +POSTHOOK: query: load data local inpath "../../data/files/test.dat" + into table ptn_t partition (emp_country="us",emp_state="ca") +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@ptn_t +POSTHOOK: Output: default@ptn_t@emp_country=us/emp_state=ca +PREHOOK: query: create table unptn_t(emp_id int comment "employee id") + stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@unptn_t +POSTHOOK: query: create table unptn_t(emp_id int comment "employee id") + stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@unptn_t +PREHOOK: query: load data local inpath "../../data/files/test.dat" + into table unptn_t +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@unptn_t +POSTHOOK: query: load data local inpath "../../data/files/test.dat" + into table unptn_t +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@unptn_t +PREHOOK: query: -- We create a table, and export the table to an export location before and after an analyze table compute statistics + +describe extended ptn_t +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ptn_t +POSTHOOK: query: -- We create a table, and export the table to an export location before and after an analyze table compute statistics + +describe extended ptn_t +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ptn_t +emp_id int employee id +emp_country string +emp_state string + +# Partition Information +# col_name data_type comment + +emp_country string +emp_state string + +#### A masked pattern was here #### +PREHOOK: query: show table extended like 'ptn_t' partition(emp_country="us",emp_state="ca") +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like 'ptn_t' partition(emp_country="us",emp_state="ca") +POSTHOOK: type: SHOW_TABLESTATUS +tableName:ptn_t +#### A masked pattern was here #### +inputformat:org.apache.hadoop.mapred.TextInputFormat +outputformat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +columns:struct columns { i32 emp_id} +partitioned:true +partitionColumns:struct partition_columns { string emp_country, string emp_state} +totalNumberFiles:1 +totalFileSize:11 +maxFileSize:11 +minFileSize:11 +#### A masked pattern was here #### + +PREHOOK: query: show create table ptn_t +PREHOOK: type: SHOW_CREATETABLE +PREHOOK: Input: default@ptn_t +POSTHOOK: query: show create table ptn_t +POSTHOOK: type: SHOW_CREATETABLE +POSTHOOK: Input: default@ptn_t +CREATE TABLE `ptn_t`( + `emp_id` int COMMENT 'employee id') +PARTITIONED BY ( + `emp_country` string, + `emp_state` string) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION +#### A masked pattern was here #### +TBLPROPERTIES ( +#### A masked pattern was here #### +PREHOOK: query: export table ptn_t to 'ql/test/data/exports/ptn_t_1' for replication('ptn_t_1') +PREHOOK: type: EXPORT +PREHOOK: Input: default@ptn_t@emp_country=us/emp_state=ca +#### A masked pattern was here #### +POSTHOOK: query: export table ptn_t to 'ql/test/data/exports/ptn_t_1' for replication('ptn_t_1') +POSTHOOK: type: EXPORT +POSTHOOK: Input: default@ptn_t@emp_country=us/emp_state=ca +#### A masked pattern was here #### +PREHOOK: query: analyze table ptn_t partition(emp_country="us",emp_state="ca") compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@ptn_t +PREHOOK: Input: default@ptn_t@emp_country=us/emp_state=ca +PREHOOK: Output: default@ptn_t +PREHOOK: Output: default@ptn_t@emp_country=us/emp_state=ca +POSTHOOK: query: analyze table ptn_t partition(emp_country="us",emp_state="ca") compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ptn_t +POSTHOOK: Input: default@ptn_t@emp_country=us/emp_state=ca +POSTHOOK: Output: default@ptn_t +POSTHOOK: Output: default@ptn_t@emp_country=us/emp_state=ca +PREHOOK: query: describe extended ptn_t +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ptn_t +POSTHOOK: query: describe extended ptn_t +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ptn_t +emp_id int employee id +emp_country string +emp_state string + +# Partition Information +# col_name data_type comment + +emp_country string +emp_state string + +#### A masked pattern was here #### +PREHOOK: query: show table extended like 'ptn_t' partition(emp_country="us",emp_state="ca") +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like 'ptn_t' partition(emp_country="us",emp_state="ca") +POSTHOOK: type: SHOW_TABLESTATUS +tableName:ptn_t +#### A masked pattern was here #### +inputformat:org.apache.hadoop.mapred.TextInputFormat +outputformat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +columns:struct columns { i32 emp_id} +partitioned:true +partitionColumns:struct partition_columns { string emp_country, string emp_state} +totalNumberFiles:1 +totalFileSize:11 +maxFileSize:11 +minFileSize:11 +#### A masked pattern was here #### + +PREHOOK: query: show create table ptn_t +PREHOOK: type: SHOW_CREATETABLE +PREHOOK: Input: default@ptn_t +POSTHOOK: query: show create table ptn_t +POSTHOOK: type: SHOW_CREATETABLE +POSTHOOK: Input: default@ptn_t +CREATE TABLE `ptn_t`( + `emp_id` int COMMENT 'employee id') +PARTITIONED BY ( + `emp_country` string, + `emp_state` string) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION +#### A masked pattern was here #### +TBLPROPERTIES ( +#### A masked pattern was here #### +PREHOOK: query: export table ptn_t to 'ql/test/data/exports/ptn_t_2' for replication('ptn_t_2') +PREHOOK: type: EXPORT +PREHOOK: Input: default@ptn_t@emp_country=us/emp_state=ca +#### A masked pattern was here #### +POSTHOOK: query: export table ptn_t to 'ql/test/data/exports/ptn_t_2' for replication('ptn_t_2') +POSTHOOK: type: EXPORT +POSTHOOK: Input: default@ptn_t@emp_country=us/emp_state=ca +#### A masked pattern was here #### +PREHOOK: query: drop table ptn_t +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ptn_t +PREHOOK: Output: default@ptn_t +POSTHOOK: query: drop table ptn_t +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ptn_t +POSTHOOK: Output: default@ptn_t +PREHOOK: query: -- Now the same for the unpartitioned table + +describe extended unptn_t +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@unptn_t +POSTHOOK: query: -- Now the same for the unpartitioned table + +describe extended unptn_t +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@unptn_t +emp_id int employee id + +#### A masked pattern was here #### +PREHOOK: query: show create table unptn_t +PREHOOK: type: SHOW_CREATETABLE +PREHOOK: Input: default@unptn_t +POSTHOOK: query: show create table unptn_t +POSTHOOK: type: SHOW_CREATETABLE +POSTHOOK: Input: default@unptn_t +CREATE TABLE `unptn_t`( + `emp_id` int COMMENT 'employee id') +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION +#### A masked pattern was here #### +TBLPROPERTIES ( + 'COLUMN_STATS_ACCURATE'='true', + 'numFiles'='1', + 'totalSize'='11', +#### A masked pattern was here #### +PREHOOK: query: export table unptn_t to 'ql/test/data/exports/unptn_t_1' for replication('unptn_t_1') +PREHOOK: type: EXPORT +PREHOOK: Input: default@unptn_t +#### A masked pattern was here #### +POSTHOOK: query: export table unptn_t to 'ql/test/data/exports/unptn_t_1' for replication('unptn_t_1') +POSTHOOK: type: EXPORT +POSTHOOK: Input: default@unptn_t +#### A masked pattern was here #### +PREHOOK: query: analyze table unptn_t compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@unptn_t +PREHOOK: Output: default@unptn_t +POSTHOOK: query: analyze table unptn_t compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@unptn_t +POSTHOOK: Output: default@unptn_t +PREHOOK: query: describe extended unptn_t +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@unptn_t +POSTHOOK: query: describe extended unptn_t +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@unptn_t +emp_id int employee id + +#### A masked pattern was here #### +PREHOOK: query: show create table unptn_t +PREHOOK: type: SHOW_CREATETABLE +PREHOOK: Input: default@unptn_t +POSTHOOK: query: show create table unptn_t +POSTHOOK: type: SHOW_CREATETABLE +POSTHOOK: Input: default@unptn_t +CREATE TABLE `unptn_t`( + `emp_id` int COMMENT 'employee id') +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION +#### A masked pattern was here #### +TBLPROPERTIES ( + 'COLUMN_STATS_ACCURATE'='true', + 'numFiles'='1', + 'numRows'='6', + 'rawDataSize'='6', + 'totalSize'='11', +#### A masked pattern was here #### +PREHOOK: query: export table unptn_t to 'ql/test/data/exports/unptn_t_2' for replication('unptn_t_2') +PREHOOK: type: EXPORT +PREHOOK: Input: default@unptn_t +#### A masked pattern was here #### +POSTHOOK: query: export table unptn_t to 'ql/test/data/exports/unptn_t_2' for replication('unptn_t_2') +POSTHOOK: type: EXPORT +POSTHOOK: Input: default@unptn_t +#### A masked pattern was here #### +PREHOOK: query: drop table unptn_t +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@unptn_t +PREHOOK: Output: default@unptn_t +POSTHOOK: query: drop table unptn_t +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@unptn_t +POSTHOOK: Output: default@unptn_t +PREHOOK: query: -- Now we import + +explain import table import_ptn_t_1 from 'ql/test/data/exports/ptn_t_1' +PREHOOK: type: IMPORT +POSTHOOK: query: -- Now we import + +explain import table import_ptn_t_1 from 'ql/test/data/exports/ptn_t_1' +POSTHOOK: type: IMPORT +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-3 depends on stages: Stage-1, Stage-2 + Stage-0 is a root stage + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Copy +#### A masked pattern was here #### + + Stage: Stage-3 + Move Operator + tables: + partition: + emp_country us + emp_state ca + replace: true + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + name: default.import_ptn_t_1 + + Stage: Stage-0 + Create Table Operator: + Create Table + columns: emp_id int employee id + input format: org.apache.hadoop.mapred.TextInputFormat +#### A masked pattern was here #### + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition columns: emp_country string, emp_state string + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde properties: + serialization.format 1 + name: import_ptn_t_1 + table properties: + repl.last.id 0 +#### A masked pattern was here #### + + Stage: Stage-2 + Add Partition Operator: +#### A masked pattern was here #### + Spec: {emp_country=us, emp_state=ca} + +PREHOOK: query: import table import_ptn_t_1 from 'ql/test/data/exports/ptn_t_1' +PREHOOK: type: IMPORT +#### A masked pattern was here #### +PREHOOK: Output: default@import_ptn_t_1 +POSTHOOK: query: import table import_ptn_t_1 from 'ql/test/data/exports/ptn_t_1' +POSTHOOK: type: IMPORT +#### A masked pattern was here #### +POSTHOOK: Output: default@import_ptn_t_1 +POSTHOOK: Output: default@import_ptn_t_1@emp_country=us/emp_state=ca +PREHOOK: query: describe extended import_ptn_t_1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@import_ptn_t_1 +POSTHOOK: query: describe extended import_ptn_t_1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@import_ptn_t_1 +emp_id int employee id +emp_country string +emp_state string + +# Partition Information +# col_name data_type comment + +emp_country string +emp_state string + +#### A masked pattern was here #### +PREHOOK: query: show table extended like 'import_ptn_t_1' partition(emp_country="us",emp_state="ca") +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like 'import_ptn_t_1' partition(emp_country="us",emp_state="ca") +POSTHOOK: type: SHOW_TABLESTATUS +tableName:import_ptn_t_1 +#### A masked pattern was here #### +inputformat:org.apache.hadoop.mapred.TextInputFormat +outputformat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +columns:struct columns { i32 emp_id} +partitioned:true +partitionColumns:struct partition_columns { string emp_country, string emp_state} +totalNumberFiles:1 +totalFileSize:11 +maxFileSize:11 +minFileSize:11 +#### A masked pattern was here #### + +PREHOOK: query: show create table import_ptn_t_1 +PREHOOK: type: SHOW_CREATETABLE +PREHOOK: Input: default@import_ptn_t_1 +POSTHOOK: query: show create table import_ptn_t_1 +POSTHOOK: type: SHOW_CREATETABLE +POSTHOOK: Input: default@import_ptn_t_1 +CREATE TABLE `import_ptn_t_1`( + `emp_id` int COMMENT 'employee id') +PARTITIONED BY ( + `emp_country` string, + `emp_state` string) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION +#### A masked pattern was here #### +TBLPROPERTIES ( + 'repl.last.id'='0', +#### A masked pattern was here #### +PREHOOK: query: explain import table import_ptn_t_2 from 'ql/test/data/exports/ptn_t_2' +PREHOOK: type: IMPORT +POSTHOOK: query: explain import table import_ptn_t_2 from 'ql/test/data/exports/ptn_t_2' +POSTHOOK: type: IMPORT +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-3 depends on stages: Stage-1, Stage-2 + Stage-0 is a root stage + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Copy +#### A masked pattern was here #### + + Stage: Stage-3 + Move Operator + tables: + partition: + emp_country us + emp_state ca + replace: true + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + name: default.import_ptn_t_2 + + Stage: Stage-0 + Create Table Operator: + Create Table + columns: emp_id int employee id + input format: org.apache.hadoop.mapred.TextInputFormat +#### A masked pattern was here #### + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition columns: emp_country string, emp_state string + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde properties: + serialization.format 1 + name: import_ptn_t_2 + table properties: + repl.last.id 0 +#### A masked pattern was here #### + + Stage: Stage-2 + Add Partition Operator: +#### A masked pattern was here #### + Spec: {emp_country=us, emp_state=ca} + +PREHOOK: query: import table import_ptn_t_2 from 'ql/test/data/exports/ptn_t_2' +PREHOOK: type: IMPORT +#### A masked pattern was here #### +PREHOOK: Output: default@import_ptn_t_2 +POSTHOOK: query: import table import_ptn_t_2 from 'ql/test/data/exports/ptn_t_2' +POSTHOOK: type: IMPORT +#### A masked pattern was here #### +POSTHOOK: Output: default@import_ptn_t_2 +POSTHOOK: Output: default@import_ptn_t_2@emp_country=us/emp_state=ca +PREHOOK: query: describe extended import_ptn_t_2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@import_ptn_t_2 +POSTHOOK: query: describe extended import_ptn_t_2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@import_ptn_t_2 +emp_id int employee id +emp_country string +emp_state string + +# Partition Information +# col_name data_type comment + +emp_country string +emp_state string + +#### A masked pattern was here #### +PREHOOK: query: show table extended like 'import_ptn_t_2' partition(emp_country="us",emp_state="ca") +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like 'import_ptn_t_2' partition(emp_country="us",emp_state="ca") +POSTHOOK: type: SHOW_TABLESTATUS +tableName:import_ptn_t_2 +#### A masked pattern was here #### +inputformat:org.apache.hadoop.mapred.TextInputFormat +outputformat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +columns:struct columns { i32 emp_id} +partitioned:true +partitionColumns:struct partition_columns { string emp_country, string emp_state} +totalNumberFiles:1 +totalFileSize:11 +maxFileSize:11 +minFileSize:11 +#### A masked pattern was here #### + +PREHOOK: query: show create table import_ptn_t_2 +PREHOOK: type: SHOW_CREATETABLE +PREHOOK: Input: default@import_ptn_t_2 +POSTHOOK: query: show create table import_ptn_t_2 +POSTHOOK: type: SHOW_CREATETABLE +POSTHOOK: Input: default@import_ptn_t_2 +CREATE TABLE `import_ptn_t_2`( + `emp_id` int COMMENT 'employee id') +PARTITIONED BY ( + `emp_country` string, + `emp_state` string) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION +#### A masked pattern was here #### +TBLPROPERTIES ( + 'repl.last.id'='0', +#### A masked pattern was here #### +PREHOOK: query: import table import_unptn_t_1 from 'ql/test/data/exports/unptn_t_1' +PREHOOK: type: IMPORT +#### A masked pattern was here #### +PREHOOK: Output: default@import_unptn_t_1 +POSTHOOK: query: import table import_unptn_t_1 from 'ql/test/data/exports/unptn_t_1' +POSTHOOK: type: IMPORT +#### A masked pattern was here #### +POSTHOOK: Output: default@import_unptn_t_1 +PREHOOK: query: describe extended import_unptn_t_1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@import_unptn_t_1 +POSTHOOK: query: describe extended import_unptn_t_1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@import_unptn_t_1 +emp_id int employee id + +#### A masked pattern was here #### +PREHOOK: query: show table extended like 'import_unptn_t_1' +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like 'import_unptn_t_1' +POSTHOOK: type: SHOW_TABLESTATUS +tableName:import_unptn_t_1 +#### A masked pattern was here #### +inputformat:org.apache.hadoop.mapred.TextInputFormat +outputformat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +columns:struct columns { i32 emp_id} +partitioned:false +partitionColumns: +totalNumberFiles:1 +totalFileSize:11 +maxFileSize:11 +minFileSize:11 +#### A masked pattern was here #### + +PREHOOK: query: show create table import_unptn_t_1 +PREHOOK: type: SHOW_CREATETABLE +PREHOOK: Input: default@import_unptn_t_1 +POSTHOOK: query: show create table import_unptn_t_1 +POSTHOOK: type: SHOW_CREATETABLE +POSTHOOK: Input: default@import_unptn_t_1 +CREATE TABLE `import_unptn_t_1`( + `emp_id` int COMMENT 'employee id') +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION +#### A masked pattern was here #### +TBLPROPERTIES ( + 'COLUMN_STATS_ACCURATE'='true', + 'numFiles'='1', + 'repl.last.id'='0', + 'totalSize'='11', +#### A masked pattern was here #### +PREHOOK: query: import table import_unptn_t_2 from 'ql/test/data/exports/unptn_t_2' +PREHOOK: type: IMPORT +#### A masked pattern was here #### +PREHOOK: Output: default@import_unptn_t_2 +POSTHOOK: query: import table import_unptn_t_2 from 'ql/test/data/exports/unptn_t_2' +POSTHOOK: type: IMPORT +#### A masked pattern was here #### +POSTHOOK: Output: default@import_unptn_t_2 +PREHOOK: query: describe extended import_unptn_t_2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@import_unptn_t_2 +POSTHOOK: query: describe extended import_unptn_t_2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@import_unptn_t_2 +emp_id int employee id + +#### A masked pattern was here #### +PREHOOK: query: show table extended like 'import_unptn_t_2' +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like 'import_unptn_t_2' +POSTHOOK: type: SHOW_TABLESTATUS +tableName:import_unptn_t_2 +#### A masked pattern was here #### +inputformat:org.apache.hadoop.mapred.TextInputFormat +outputformat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +columns:struct columns { i32 emp_id} +partitioned:false +partitionColumns: +totalNumberFiles:1 +totalFileSize:11 +maxFileSize:11 +minFileSize:11 +#### A masked pattern was here #### + +PREHOOK: query: show create table import_unptn_t_2 +PREHOOK: type: SHOW_CREATETABLE +PREHOOK: Input: default@import_unptn_t_2 +POSTHOOK: query: show create table import_unptn_t_2 +POSTHOOK: type: SHOW_CREATETABLE +POSTHOOK: Input: default@import_unptn_t_2 +CREATE TABLE `import_unptn_t_2`( + `emp_id` int COMMENT 'employee id') +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION +#### A masked pattern was here #### +TBLPROPERTIES ( + 'COLUMN_STATS_ACCURATE'='true', + 'numFiles'='1', + 'numRows'='6', + 'rawDataSize'='6', + 'repl.last.id'='0', + 'totalSize'='11', +#### A masked pattern was here #### +PREHOOK: query: -- clean up + +drop table import_ptn_t_1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@import_ptn_t_1 +PREHOOK: Output: default@import_ptn_t_1 +POSTHOOK: query: -- clean up + +drop table import_ptn_t_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@import_ptn_t_1 +POSTHOOK: Output: default@import_ptn_t_1 +PREHOOK: query: drop table import_ptn_t_2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@import_ptn_t_2 +PREHOOK: Output: default@import_ptn_t_2 +POSTHOOK: query: drop table import_ptn_t_2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@import_ptn_t_2 +POSTHOOK: Output: default@import_ptn_t_2 +PREHOOK: query: drop table import_unptn_t_1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@import_unptn_t_1 +PREHOOK: Output: default@import_unptn_t_1 +POSTHOOK: query: drop table import_unptn_t_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@import_unptn_t_1 +POSTHOOK: Output: default@import_unptn_t_1 +PREHOOK: query: drop table import_unptn_t_2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@import_unptn_t_2 +PREHOOK: Output: default@import_unptn_t_2 +POSTHOOK: query: drop table import_unptn_t_2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@import_unptn_t_2 +POSTHOOK: Output: default@import_unptn_t_2