diff --git a/data/files/repl_dump/test_hcube_2/tbl6/_metadata b/data/files/repl_dump/test_hcube_2/tbl6/_metadata index 75ead6a3db..16f5c7ea76 100644 --- a/data/files/repl_dump/test_hcube_2/tbl6/_metadata +++ b/data/files/repl_dump/test_hcube_2/tbl6/_metadata @@ -1 +1 @@ -{"version":"0.2","repl.scope":"all","repl.event.id":"replv2","repl.last.id":"548434","repl.noop":"false","repl.lazy":"true","repl.is.replace":"true","repl.valid.writeid.list":"test_hcube_2.tbl6:1:9223372036854775807::","repl.valid.txnid.list":"1915:9223372036854775807::1899,1907","table":"{\"1\":{\"str\":\"tbl6\"},\"2\":{\"str\":\"test_hcube_2\"},\"3\":{\"str\":\"hive\"},\"4\":{\"i32\":1569308637},\"5\":{\"i32\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"lst\":[\"rec\",1,{\"1\":{\"str\":\"fld\"},\"2\":{\"str\":\"int\"}}]},\"2\":{\"str\":\"hdfs://hdpcluster-0/warehouse/tablespace/managed/hive/test_hcube_2.db/tbl6\"},\"3\":{\"str\":\"org.apache.hadoop.hive.ql.io.orc.OrcInputFormat\"},\"4\":{\"str\":\"org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat\"},\"5\":{\"tf\":0},\"6\":{\"i32\":-1},\"7\":{\"rec\":{\"2\":{\"str\":\"org.apache.hadoop.hive.ql.io.orc.OrcSerde\"},\"3\":{\"map\":[\"str\",\"str\",1,{\"serialization.format\":\"1\"}]}}},\"8\":{\"lst\":[\"str\",0]},\"9\":{\"lst\":[\"rec\",0]},\"10\":{\"map\":[\"str\",\"str\",0,{}]},\"11\":{\"rec\":{\"1\":{\"lst\":[\"str\",0]},\"2\":{\"lst\":[\"lst\",0]},\"3\":{\"map\":[\"lst\",\"str\",0,{}]}}},\"12\":{\"tf\":0}}},\"8\":{\"lst\":[\"rec\",0]},\"9\":{\"map\":[\"str\",\"str\",10,{\"totalSize\":\"611\",\"numRows\":\"1\",\"rawDataSize\":\"0\",\"transactional_properties\":\"default\",\"COLUMN_STATS_ACCURATE\":\"{\\\"BASIC_STATS\\\":\\\"true\\\",\\\"COLUMN_STATS\\\":{\\\"fld\\\":\\\"true\\\"}}\",\"numFiles\":\"1\",\"transient_lastDdlTime\":\"1569308979\",\"bucketing_version\":\"2\",\"repl.last.id\":\"548434\",\"transactional\":\"true\"}]},\"12\":{\"str\":\"MANAGED_TABLE\"},\"15\":{\"tf\":0},\"17\":{\"str\":\"hive\"},\"18\":{\"i32\":1},\"19\":{\"i64\":1},\"21\":{\"rec\":{\"1\":{\"rec\":{\"1\":{\"tf\":1},\"2\":{\"str\":\"test_hcube_2\"},\"3\":{\"str\":\"tbl6\"},\"5\":{\"i64\":1569308979},\"6\":{\"str\":\"hive\"}}},\"2\":{\"lst\":[\"rec\",1,{\"1\":{\"str\":\"fld\"},\"2\":{\"str\":\"int\"},\"3\":{\"rec\":{\"2\":{\"rec\":{\"1\":{\"i64\":1},\"2\":{\"i64\":1},\"3\":{\"i64\":0},\"4\":{\"i64\":1}}}}}}]}}}}","partitions":[]} \ No newline at end of file +{"version":"0.2","repl.scope":"all","repl.event.id":"replv2","repl.last.id":"326198","repl.noop":"false","repl.lazy":"true","repl.is.replace":"true","repl.valid.writeid.list":"test_hcube_2.tbl6:2:9223372036854775807::","repl.valid.txnid.list":"162684:9223372036854775807::","table":"{\"1\":{\"str\":\"tbl6\"},\"2\":{\"str\":\"test_hcube_2\"},\"3\":{\"str\":\"hive\"},\"4\":{\"i32\":1570685217},\"5\":{\"i32\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"lst\":[\"rec\",1,{\"1\":{\"str\":\"fld\"},\"2\":{\"str\":\"int\"}}]},\"2\":{\"str\":\"hdfs://ctr-e141-1563959304486-40246-01-000007.hwx.site:8020/warehouse/tablespace/managed/hive/test_hcube_2.db/tbl6\"},\"3\":{\"str\":\"org.apache.hadoop.hive.ql.io.orc.OrcInputFormat\"},\"4\":{\"str\":\"org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat\"},\"5\":{\"tf\":0},\"6\":{\"i32\":-1},\"7\":{\"rec\":{\"2\":{\"str\":\"org.apache.hadoop.hive.ql.io.orc.OrcSerde\"},\"3\":{\"map\":[\"str\",\"str\",1,{\"serialization.format\":\"1\"}]}}},\"8\":{\"lst\":[\"str\",0]},\"9\":{\"lst\":[\"rec\",0]},\"10\":{\"map\":[\"str\",\"str\",0,{}]},\"11\":{\"rec\":{\"1\":{\"lst\":[\"str\",0]},\"2\":{\"lst\":[\"lst\",0]},\"3\":{\"map\":[\"lst\",\"str\",0,{}]}}},\"12\":{\"tf\":0}}},\"8\":{\"lst\":[\"rec\",1,{\"1\":{\"str\":\"fld1\"},\"2\":{\"str\":\"int\"}}]},\"9\":{\"map\":[\"str\",\"str\",5,{\"transient_lastDdlTime\":\"1570685217\",\"bucketing_version\":\"2\",\"repl.last.id\":\"326198\",\"transactional_properties\":\"default\",\"transactional\":\"true\"}]},\"12\":{\"str\":\"MANAGED_TABLE\"},\"15\":{\"tf\":0},\"17\":{\"str\":\"hive\"},\"18\":{\"i32\":1},\"19\":{\"i64\":0}}","partitions":["{\"1\":{\"lst\":[\"str\",1,\"1\"]},\"2\":{\"str\":\"test_hcube_2\"},\"3\":{\"str\":\"tbl6\"},\"4\":{\"i32\":1570685277},\"5\":{\"i32\":0},\"6\":{\"rec\":{\"1\":{\"lst\":[\"rec\",1,{\"1\":{\"str\":\"fld\"},\"2\":{\"str\":\"int\"}}]},\"2\":{\"str\":\"hdfs://ctr-e141-1563959304486-40246-01-000007.hwx.site:8020/warehouse/tablespace/managed/hive/test_hcube_2.db/tbl6/fld1=1\"},\"3\":{\"str\":\"org.apache.hadoop.hive.ql.io.orc.OrcInputFormat\"},\"4\":{\"str\":\"org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat\"},\"5\":{\"tf\":0},\"6\":{\"i32\":-1},\"7\":{\"rec\":{\"2\":{\"str\":\"org.apache.hadoop.hive.ql.io.orc.OrcSerde\"},\"3\":{\"map\":[\"str\",\"str\",1,{\"serialization.format\":\"1\"}]}}},\"8\":{\"lst\":[\"str\",0]},\"9\":{\"lst\":[\"rec\",0]},\"10\":{\"map\":[\"str\",\"str\",0,{}]},\"11\":{\"rec\":{\"1\":{\"lst\":[\"str\",0]},\"2\":{\"lst\":[\"lst\",0]},\"3\":{\"map\":[\"lst\",\"str\",0,{}]}}},\"12\":{\"tf\":0}}},\"7\":{\"map\":[\"str\",\"str\",7,{\"totalSize\":\"611\",\"numRows\":\"1\",\"rawDataSize\":\"0\",\"COLUMN_STATS_ACCURATE\":\"{\\\"BASIC_STATS\\\":\\\"true\\\",\\\"COLUMN_STATS\\\":{\\\"fld\\\":\\\"true\\\"}}\",\"numFiles\":\"1\",\"transient_lastDdlTime\":\"1570685277\",\"repl.last.id\":\"326198\"}]},\"9\":{\"str\":\"hive\"},\"10\":{\"i64\":1},\"12\":{\"rec\":{\"1\":{\"rec\":{\"1\":{\"tf\":0},\"2\":{\"str\":\"test_hcube_2\"},\"3\":{\"str\":\"tbl6\"},\"4\":{\"str\":\"fld1=1\"},\"5\":{\"i64\":1570685277},\"6\":{\"str\":\"hive\"}}},\"2\":{\"lst\":[\"rec\",1,{\"1\":{\"str\":\"fld\"},\"2\":{\"str\":\"int\"},\"3\":{\"rec\":{\"2\":{\"rec\":{\"1\":{\"i64\":1},\"2\":{\"i64\":1},\"3\":{\"i64\":0},\"4\":{\"i64\":1}}}}}}]}}}}","{\"1\":{\"lst\":[\"str\",1,\"2\"]},\"2\":{\"str\":\"test_hcube_2\"},\"3\":{\"str\":\"tbl6\"},\"4\":{\"i32\":1570685286},\"5\":{\"i32\":0},\"6\":{\"rec\":{\"1\":{\"lst\":[\"rec\",1,{\"1\":{\"str\":\"fld\"},\"2\":{\"str\":\"int\"}}]},\"2\":{\"str\":\"hdfs://ctr-e141-1563959304486-40246-01-000007.hwx.site:8020/warehouse/tablespace/managed/hive/test_hcube_2.db/tbl6/fld1=2\"},\"3\":{\"str\":\"org.apache.hadoop.hive.ql.io.orc.OrcInputFormat\"},\"4\":{\"str\":\"org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat\"},\"5\":{\"tf\":0},\"6\":{\"i32\":-1},\"7\":{\"rec\":{\"2\":{\"str\":\"org.apache.hadoop.hive.ql.io.orc.OrcSerde\"},\"3\":{\"map\":[\"str\",\"str\",1,{\"serialization.format\":\"1\"}]}}},\"8\":{\"lst\":[\"str\",0]},\"9\":{\"lst\":[\"rec\",0]},\"10\":{\"map\":[\"str\",\"str\",0,{}]},\"11\":{\"rec\":{\"1\":{\"lst\":[\"str\",0]},\"2\":{\"lst\":[\"lst\",0]},\"3\":{\"map\":[\"lst\",\"str\",0,{}]}}},\"12\":{\"tf\":0}}},\"7\":{\"map\":[\"str\",\"str\",7,{\"totalSize\":\"620\",\"numRows\":\"1\",\"rawDataSize\":\"0\",\"COLUMN_STATS_ACCURATE\":\"{\\\"BASIC_STATS\\\":\\\"true\\\",\\\"COLUMN_STATS\\\":{\\\"fld\\\":\\\"true\\\"}}\",\"numFiles\":\"1\",\"transient_lastDdlTime\":\"1570685286\",\"repl.last.id\":\"326198\"}]},\"9\":{\"str\":\"hive\"},\"10\":{\"i64\":2},\"12\":{\"rec\":{\"1\":{\"rec\":{\"1\":{\"tf\":0},\"2\":{\"str\":\"test_hcube_2\"},\"3\":{\"str\":\"tbl6\"},\"4\":{\"str\":\"fld1=2\"},\"5\":{\"i64\":1570685286},\"6\":{\"str\":\"hive\"}}},\"2\":{\"lst\":[\"rec\",1,{\"1\":{\"str\":\"fld\"},\"2\":{\"str\":\"int\"},\"3\":{\"rec\":{\"2\":{\"rec\":{\"1\":{\"i64\":1},\"2\":{\"i64\":1},\"3\":{\"i64\":0},\"4\":{\"i64\":1}}}}}}]}}}}"]} diff --git a/data/files/repl_dump/test_hcube_2/tbl6/data/_files b/data/files/repl_dump/test_hcube_2/tbl6/fld1=1/_files similarity index 100% rename from data/files/repl_dump/test_hcube_2/tbl6/data/_files rename to data/files/repl_dump/test_hcube_2/tbl6/fld1=1/_files diff --git a/data/files/repl_dump/test_hcube_2/tbl6/fld1=2/_files b/data/files/repl_dump/test_hcube_2/tbl6/fld1=2/_files new file mode 100644 index 0000000000..7101dcf4ad --- /dev/null +++ b/data/files/repl_dump/test_hcube_2/tbl6/fld1=2/_files @@ -0,0 +1 @@ +../../data/files/test_hcube_2.db/tbl6/delta_0000001_0000001_0000/bucket_00000###delta_0000001_0000001_0000 diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/AlterTableAddPartitionOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/AlterTableAddPartitionOperation.java index d8597a888c..0adccf6740 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/AlterTableAddPartitionOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/AlterTableAddPartitionOperation.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.common.ValidReaderWriteIdList; import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; @@ -128,6 +129,11 @@ private Partition convertPartitionSpecToMetaPartition(Table table, } if (partitionSpec.getColStats() != null) { partition.setColStats(partitionSpec.getColStats()); + + ColumnStatistics statistics = partition.getColStats(); + if (statistics != null && statistics.getEngine() == null) { + statistics.setEngine(org.apache.hadoop.hive.conf.Constants.HIVE_ENGINE); + } // Statistics will have an associated write Id for a transactional table. We need it to update column statistics. partition.setWriteId(partitionSpec.getWriteId()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TableHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TableHandler.java index 266d034e25..f6d57e86f5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TableHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TableHandler.java @@ -77,7 +77,6 @@ x.setEventType(eventType); // REPL LOAD is not partition level. It is always DB or table level. So, passing null for partition specs. - // Also, REPL LOAD doesn't support external table and hence no location set as well. ImportSemanticAnalyzer.prepareImport(false, isLocationSet, isExternal, false, (context.precursor != null), parsedLocation, null, context.dbName, null, context.location, x, updatedMetadata, context.getTxnMgr(), tuple.writeId); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/UpdatePartColStatHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/UpdatePartColStatHandler.java index d3d8d12759..8b1741e2ae 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/UpdatePartColStatHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/UpdatePartColStatHandler.java @@ -40,6 +40,10 @@ // Update tablename and database name in the statistics object ColumnStatistics colStats = upcsm.getColumnStatistics(); + // In older version of hive, engine might not have set. + if (colStats.getEngine() == null) { + colStats.setEngine(org.apache.hadoop.hive.conf.Constants.HIVE_ENGINE); + } ColumnStatisticsDesc colStatsDesc = colStats.getStatsDesc(); if (!context.isDbNameEmpty()) { colStatsDesc.setDbName(context.dbName); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/UpdateTableColStatHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/UpdateTableColStatHandler.java index 139c50f822..6c54f9753a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/UpdateTableColStatHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/UpdateTableColStatHandler.java @@ -40,6 +40,10 @@ // Update tablename and database name in the statistics object ColumnStatistics colStats = utcsm.getColumnStatistics(); + // In older version of hive, engine might not have set. + if (colStats.getEngine() == null) { + colStats.setEngine(org.apache.hadoop.hive.conf.Constants.HIVE_ENGINE); + } ColumnStatisticsDesc colStatsDesc = colStats.getStatsDesc(); colStatsDesc.setDbName(context.dbName); if (!context.isDbNameEmpty()) { diff --git a/ql/src/test/queries/clientpositive/repl_load_old_version.q b/ql/src/test/queries/clientpositive/repl_load_old_version.q index b01dc4a49b..11ed75dcce 100644 --- a/ql/src/test/queries/clientpositive/repl_load_old_version.q +++ b/ql/src/test/queries/clientpositive/repl_load_old_version.q @@ -7,4 +7,4 @@ select * from tbl2 order by fld; select * from tbl3 order by fld; select * from tbl4 order by fld; select * from tbl5 order by fld; -select * from tbl6 order by fld; +select * from tbl6 order by fld1; diff --git a/ql/src/test/results/clientpositive/repl_load_old_version.q.out b/ql/src/test/results/clientpositive/repl_load_old_version.q.out index 9ba534f199..77116d1a91 100644 --- a/ql/src/test/results/clientpositive/repl_load_old_version.q.out +++ b/ql/src/test/results/clientpositive/repl_load_old_version.q.out @@ -69,12 +69,17 @@ POSTHOOK: type: QUERY POSTHOOK: Input: test_db@tbl5 #### A masked pattern was here #### 1 -PREHOOK: query: select * from tbl6 order by fld +PREHOOK: query: select * from tbl6 order by fld1 PREHOOK: type: QUERY PREHOOK: Input: test_db@tbl6 +PREHOOK: Input: test_db@tbl6@fld1=1 +PREHOOK: Input: test_db@tbl6@fld1=2 #### A masked pattern was here #### -POSTHOOK: query: select * from tbl6 order by fld +POSTHOOK: query: select * from tbl6 order by fld1 POSTHOOK: type: QUERY POSTHOOK: Input: test_db@tbl6 +POSTHOOK: Input: test_db@tbl6@fld1=1 +POSTHOOK: Input: test_db@tbl6@fld1=2 #### A masked pattern was here #### -1 +1 1 +1 2