diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 74e3c8a..8e9d6af 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -783,6 +783,13 @@ HIVEMERGERCFILEBLOCKLEVEL("hive.merge.rcfile.block.level", true, ""), HIVEMERGEINPUTFORMATBLOCKLEVEL("hive.merge.input.format.block.level", "org.apache.hadoop.hive.ql.io.rcfile.merge.RCFileBlockMergeInputFormat", ""), + HIVEMERGEORCFILESTRIPELEVEL("hive.merge.orcfile.stripe.level", true, + "When hive.merge.mapfiles or hive merge.mapredfiles is enabled while writing a\n" + + " table with ORC file format, enabling this config will do stripe level fast merge\n" + + " for small orc files. Note that enabling this config will not honor padding tolerance\n" + + " config (hive.exec.orc.block.padding.tolerance)."), + HIVEMERGEINPUTFORMATSTRIPELEVEL("hive.merge.input.format.stripe.level", + "org.apache.hadoop.hive.ql.io.orc.OrcFileStripeMergeInputFormat", ""), HIVEMERGECURRENTJOBHASDYNAMICPARTITIONS( "hive.merge.current.job.has.dynamic.partitions", false, ""), diff --git conf/hive-default.xml.template conf/hive-default.xml.template index d6a8e70..ad33634 100644 --- conf/hive-default.xml.template +++ conf/hive-default.xml.template @@ -1266,6 +1266,21 @@ + hive.merge.orcfile.stripe.level + true + + When hive.merge.mapfiles or hive merge.mapredfiles is enabled while writing a + table with ORC file format, enabling this config will do stripe level fast merge + for small orc files. Note that enabling this config will not honor padding tolerance + config (hive.exec.orc.block.padding.tolerance). + + + + hive.merge.input.format.stripe.level + org.apache.hadoop.hive.ql.io.orc.OrcFileStripeMergeInputFormat + + + hive.merge.current.job.has.dynamic.partitions false diff --git itests/qtest/testconfiguration.properties itests/qtest/testconfiguration.properties index 6a3ee1d..bc1fb91 100644 --- itests/qtest/testconfiguration.properties +++ itests/qtest/testconfiguration.properties @@ -2,4 +2,5 @@ minimr.query.files=stats_counter_partitioned.q,list_bucket_dml_10.q,input16_cc.q minimr.query.negative.files=cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q,file_with_header_footer_negative.q,udf_local_resource.q minitez.query.files=tez_fsstat.q,mapjoin_decimal.q,tez_join_tests.q,tez_joins_explain.q,mrr.q,tez_dml.q,tez_insert_overwrite_local_directory_1.q,tez_union.q,bucket_map_join_tez1.q,bucket_map_join_tez2.q,tez_schema_evolution.q,tez_join_hash.q minitez.query.files.shared=cross_product_check_1.q,cross_product_check_2.q,dynpart_sort_opt_vectorization.q,dynpart_sort_optimization.q,orc_analyze.q,join0.q,join1.q,auto_join0.q,auto_join1.q,bucket2.q,bucket3.q,bucket4.q,count.q,create_merge_compressed.q,cross_join.q,ctas.q,custom_input_output_format.q,disable_merge_for_bucketing.q,enforce_order.q,filter_join_breaktask.q,filter_join_breaktask2.q,groupby1.q,groupby2.q,groupby3.q,having.q,insert1.q,insert_into1.q,insert_into2.q,leftsemijoin.q,limit_pushdown.q,load_dyn_part1.q,load_dyn_part2.q,load_dyn_part3.q,mapjoin_mapjoin.q,mapreduce1.q,mapreduce2.q,merge1.q,merge2.q,metadata_only_queries.q,sample1.q,subquery_in.q,subquery_exists.q,vectorization_15.q,ptf.q,stats_counter.q,stats_noscan_1.q,stats_counter_partitioned.q,union2.q,union3.q,union4.q,union5.q,union6.q,union7.q,union8.q,union9.q,transform1.q,transform2.q,transform_ppr1.q,transform_ppr2.q,script_env_var1.q,script_env_var2.q,script_pipe.q,scriptfile1.q,metadataonly1.q,temp_table.q,vectorized_ptf.q,optimize_nullscan.q,vector_cast_constant.q +minitez.query.files.shared=orc_merge1.q,orc_merge2.q,orc_merge3.q,orc_merge4.q,alter_merge_orc.q,alter_merge_2_orc.q,alter_merge_stats_orc.q,cross_product_check_1.q,cross_product_check_2.q,dynpart_sort_opt_vectorization.q,dynpart_sort_optimization.q,orc_analyze.q,join0.q,join1.q,auto_join0.q,auto_join1.q,bucket2.q,bucket3.q,bucket4.q,count.q,create_merge_compressed.q,cross_join.q,ctas.q,custom_input_output_format.q,disable_merge_for_bucketing.q,enforce_order.q,filter_join_breaktask.q,filter_join_breaktask2.q,groupby1.q,groupby2.q,groupby3.q,having.q,insert1.q,insert_into1.q,insert_into2.q,leftsemijoin.q,limit_pushdown.q,load_dyn_part1.q,load_dyn_part2.q,load_dyn_part3.q,mapjoin_mapjoin.q,mapreduce1.q,mapreduce2.q,merge1.q,merge2.q,metadata_only_queries.q,sample1.q,subquery_in.q,subquery_exists.q,vectorization_15.q,ptf.q,stats_counter.q,stats_noscan_1.q,stats_counter_partitioned.q,union2.q,union3.q,union4.q,union5.q,union6.q,union7.q,union8.q,union9.q,transform1.q,transform2.q,transform_ppr1.q,transform_ppr2.q,script_env_var1.q,script_env_var2.q,script_pipe.q,scriptfile1.q,metadataonly1.q,temp_table.q,vectorized_ptf.q,optimize_nullscan.q beeline.positive.exclude=add_part_exist.q,alter1.q,alter2.q,alter4.q,alter5.q,alter_rename_partition.q,alter_rename_partition_authorization.q,archive.q,archive_corrupt.q,archive_multi.q,archive_mr_1806.q,archive_multi_mr_1806.q,authorization_1.q,authorization_2.q,authorization_4.q,authorization_5.q,authorization_6.q,authorization_7.q,ba_table1.q,ba_table2.q,ba_table3.q,ba_table_udfs.q,binary_table_bincolserde.q,binary_table_colserde.q,cluster.q,columnarserde_create_shortcut.q,combine2.q,constant_prop.q,create_nested_type.q,create_or_replace_view.q,create_struct_table.q,create_union_table.q,database.q,database_location.q,database_properties.q,ddltime.q,describe_database_json.q,drop_database_removes_partition_dirs.q,escape1.q,escape2.q,exim_00_nonpart_empty.q,exim_01_nonpart.q,exim_02_00_part_empty.q,exim_02_part.q,exim_03_nonpart_over_compat.q,exim_04_all_part.q,exim_04_evolved_parts.q,exim_05_some_part.q,exim_06_one_part.q,exim_07_all_part_over_nonoverlap.q,exim_08_nonpart_rename.q,exim_09_part_spec_nonoverlap.q,exim_10_external_managed.q,exim_11_managed_external.q,exim_12_external_location.q,exim_13_managed_location.q,exim_14_managed_location_over_existing.q,exim_15_external_part.q,exim_16_part_external.q,exim_17_part_managed.q,exim_18_part_external.q,exim_19_00_part_external_location.q,exim_19_part_external_location.q,exim_20_part_managed_location.q,exim_21_export_authsuccess.q,exim_22_import_exist_authsuccess.q,exim_23_import_part_authsuccess.q,exim_24_import_nonexist_authsuccess.q,global_limit.q,groupby_complex_types.q,groupby_complex_types_multi_single_reducer.q,index_auth.q,index_auto.q,index_auto_empty.q,index_bitmap.q,index_bitmap1.q,index_bitmap2.q,index_bitmap3.q,index_bitmap_auto.q,index_bitmap_rc.q,index_compact.q,index_compact_1.q,index_compact_2.q,index_compact_3.q,index_stale_partitioned.q,init_file.q,input16.q,input16_cc.q,input46.q,input_columnarserde.q,input_dynamicserde.q,input_lazyserde.q,input_testxpath3.q,input_testxpath4.q,insert2_overwrite_partitions.q,insertexternal1.q,join_thrift.q,lateral_view.q,load_binary_data.q,load_exist_part_authsuccess.q,load_nonpart_authsuccess.q,load_part_authsuccess.q,loadpart_err.q,lock1.q,lock2.q,lock3.q,lock4.q,merge_dynamic_partition.q,multi_insert.q,multi_insert_move_tasks_share_dependencies.q,null_column.q,ppd_clusterby.q,query_with_semi.q,rename_column.q,sample6.q,sample_islocalmode_hook.q,set_processor_namespaces.q,show_tables.q,source.q,split_sample.q,str_to_map.q,transform1.q,udaf_collect_set.q,udaf_context_ngrams.q,udaf_histogram_numeric.q,udaf_ngrams.q,udaf_percentile_approx.q,udf_array.q,udf_bitmap_and.q,udf_bitmap_or.q,udf_explode.q,udf_format_number.q,udf_map.q,udf_map_keys.q,udf_map_values.q,udf_max.q,udf_min.q,udf_named_struct.q,udf_percentile.q,udf_printf.q,udf_sentences.q,udf_sort_array.q,udf_split.q,udf_struct.q,udf_substr.q,udf_translate.q,udf_union.q,udf_xpath.q,udtf_stack.q,view.q,virtual_column.q diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index c80a2a3..bf56d8f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -89,7 +89,7 @@ import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; -import org.apache.hadoop.hive.ql.io.rcfile.merge.BlockMergeTask; +import org.apache.hadoop.hive.ql.io.rcfile.merge.MergeTask; import org.apache.hadoop.hive.ql.io.rcfile.merge.MergeWork; import org.apache.hadoop.hive.ql.io.rcfile.truncate.ColumnTruncateTask; import org.apache.hadoop.hive.ql.io.rcfile.truncate.ColumnTruncateWork; @@ -550,12 +550,13 @@ private int mergeFiles(Hive db, AlterTablePartMergeFilesDesc mergeFilesDesc) throws HiveException { // merge work only needs input and output. MergeWork mergeWork = new MergeWork(mergeFilesDesc.getInputDir(), - mergeFilesDesc.getOutputDir()); + mergeFilesDesc.getOutputDir(), mergeFilesDesc.getInputFormatClass()); mergeWork.setListBucketingCtx(mergeFilesDesc.getLbCtx()); mergeWork.resolveConcatenateMerge(db.getConf()); mergeWork.setMapperCannotSpanPartns(true); + mergeWork.setSourceTableInputFormat(mergeFilesDesc.getInputFormatClass()); DriverContext driverCxt = new DriverContext(); - BlockMergeTask taskExec = new BlockMergeTask(); + MergeTask taskExec = new MergeTask(); taskExec.initialize(db.getConf(), null, driverCxt); taskExec.setWork(mergeWork); taskExec.setQueryPlan(this.getQueryPlan()); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java index e1dc911..fa4f46b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java @@ -36,7 +36,7 @@ import org.apache.hadoop.hive.ql.hooks.LineageInfo.DataContainer; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils; -import org.apache.hadoop.hive.ql.io.rcfile.merge.BlockMergeTask; +import org.apache.hadoop.hive.ql.io.rcfile.merge.MergeTask; import org.apache.hadoop.hive.ql.lockmgr.HiveLock; import org.apache.hadoop.hive.ql.lockmgr.HiveLockManager; import org.apache.hadoop.hive.ql.lockmgr.HiveLockObj; @@ -294,7 +294,7 @@ public int execute(DriverContext driverContext) { while (task.getParentTasks() != null && task.getParentTasks().size() == 1) { task = (Task)task.getParentTasks().get(0); // If it was a merge task or a local map reduce task, nothing can be inferred - if (task instanceof BlockMergeTask || task instanceof MapredLocalTask) { + if (task instanceof MergeTask || task instanceof MapredLocalTask) { break; } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java index ad6e19c..eae80c0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java @@ -28,7 +28,7 @@ import org.apache.hadoop.hive.ql.exec.tez.TezTask; import org.apache.hadoop.hive.ql.index.IndexMetadataChangeTask; import org.apache.hadoop.hive.ql.index.IndexMetadataChangeWork; -import org.apache.hadoop.hive.ql.io.rcfile.merge.BlockMergeTask; +import org.apache.hadoop.hive.ql.io.rcfile.merge.MergeTask; import org.apache.hadoop.hive.ql.io.rcfile.merge.MergeWork; import org.apache.hadoop.hive.ql.io.rcfile.stats.PartialScanTask; import org.apache.hadoop.hive.ql.io.rcfile.stats.PartialScanWork; @@ -93,7 +93,7 @@ public TaskTuple(Class workClass, Class> taskClass) { taskvec.add(new TaskTuple(StatsNoJobWork.class, StatsNoJobTask.class)); taskvec.add(new TaskTuple(ColumnStatsWork.class, ColumnStatsTask.class)); taskvec.add(new TaskTuple(MergeWork.class, - BlockMergeTask.class)); + MergeTask.class)); taskvec.add(new TaskTuple(DependencyCollectionWork.class, DependencyCollectionTask.class)); taskvec.add(new TaskTuple(PartialScanWork.class, diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 10ae157..81db284 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -120,6 +120,7 @@ import org.apache.hadoop.hive.ql.io.OneNullRowInputFormat; import org.apache.hadoop.hive.ql.io.RCFile; import org.apache.hadoop.hive.ql.io.ReworkMapredInputFormat; +import org.apache.hadoop.hive.ql.io.orc.OrcFileMergeMapper; import org.apache.hadoop.hive.ql.io.rcfile.merge.MergeWork; import org.apache.hadoop.hive.ql.io.rcfile.merge.RCFileMergeMapper; import org.apache.hadoop.hive.ql.io.rcfile.stats.PartialScanMapper; @@ -346,7 +347,8 @@ private static BaseWork getBaseWork(Configuration conf, String name) { if(MAP_PLAN_NAME.equals(name)){ if (ExecMapper.class.getName().equals(conf.get(MAPRED_MAPPER_CLASS))){ gWork = deserializePlan(in, MapWork.class, conf); - } else if(RCFileMergeMapper.class.getName().equals(conf.get(MAPRED_MAPPER_CLASS))) { + } else if(RCFileMergeMapper.class.getName().equals(conf.get(MAPRED_MAPPER_CLASS)) || + OrcFileMergeMapper.class.getName().equals(conf.get(MAPRED_MAPPER_CLASS))) { gWork = deserializePlan(in, MergeWork.class, conf); } else if(ColumnTruncateMapper.class.getName().equals(conf.get(MAPRED_MAPPER_CLASS))) { gWork = deserializePlan(in, ColumnTruncateWork.class, conf); diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileKeyWrapper.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileKeyWrapper.java new file mode 100644 index 0000000..6eb0e22 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileKeyWrapper.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.io.orc.OrcProto; +import org.apache.hadoop.io.WritableComparable; + +/** + * Key for OrcFileMergeMapper task. Contains orc file related information that + * should match before merging two orc files. + */ +public class OrcFileKeyWrapper implements WritableComparable { + + protected Path inputPath; + protected CompressionKind compression; + protected long compressBufferSize; + protected List types; + protected int rowIndexStride; + protected List versionList; + + public List getVersionList() { + return versionList; + } + + public void setVersionList(List versionList) { + this.versionList = versionList; + } + + public int getRowIndexStride() { + return rowIndexStride; + } + + public void setRowIndexStride(int rowIndexStride) { + this.rowIndexStride = rowIndexStride; + } + + public long getCompressBufferSize() { + return compressBufferSize; + } + + public void setCompressBufferSize(long compressBufferSize) { + this.compressBufferSize = compressBufferSize; + } + + public CompressionKind getCompression() { + return compression; + } + + public void setCompression(CompressionKind compression) { + this.compression = compression; + } + + public List getTypes() { + return types; + } + + public void setTypes(List types) { + this.types = types; + } + + public Path getInputPath() { + return inputPath; + } + + public void setInputPath(Path inputPath) { + this.inputPath = inputPath; + } + + @Override + public void write(DataOutput out) throws IOException { + throw new RuntimeException("Not supported."); + } + + @Override + public void readFields(DataInput in) throws IOException { + throw new RuntimeException("Not supported."); + } + + @Override + public int compareTo(OrcFileKeyWrapper o) { + return inputPath.compareTo(o.inputPath); + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileMergeMapper.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileMergeMapper.java new file mode 100644 index 0000000..4876913 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileMergeMapper.java @@ -0,0 +1,351 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.io.rcfile.merge.MergeOutputFormat; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.shims.CombineHiveKey; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.MapReduceBase; +import org.apache.hadoop.mapred.Mapper; +import org.apache.hadoop.mapred.OutputCollector; +import org.apache.hadoop.mapred.Reporter; + +/** + * Map task fast merging of ORC files. + */ +public class OrcFileMergeMapper extends MapReduceBase implements + Mapper { + + private JobConf jc; + Class outputClass; + Writer outWriter; + + Path finalPath; + FileSystem fs; + + boolean exception = false; + boolean autoDelete = false; + Path outPath; + + // These parameters must match for all orc files involved in merging + CompressionKind compression = null; + long compressBuffSize = 0; + List version; + int columnCount = 0; + int rowIndexStride = 0; + + boolean hasDynamicPartitions = false; + boolean isListBucketingDML = false; + boolean isListBucketingAlterTableConcatenate = false; + + // used as depth for dir-calculation and if it is list bucketing case. + int listBucketingDepth; + boolean tmpPathFixedConcatenate = false; + boolean tmpPathFixed = false; + Path tmpPath; + Path taskTmpPath; + Path dpPath; + private byte[] buffer; + Path prevPath; + private Reader reader; + private FSDataInputStream fdis; + + public final static Log LOG = LogFactory.getLog("OrcFileMergeMapper"); + + @Override + public void configure(JobConf job) { + jc = job; + hasDynamicPartitions = HiveConf.getBoolVar(job, + HiveConf.ConfVars.HIVEMERGECURRENTJOBHASDYNAMICPARTITIONS); + isListBucketingAlterTableConcatenate = HiveConf.getBoolVar(job, + HiveConf.ConfVars.HIVEMERGECURRENTJOBCONCATENATELISTBUCKETING); + listBucketingDepth = HiveConf.getIntVar(job, + HiveConf.ConfVars.HIVEMERGECURRENTJOBCONCATENATELISTBUCKETINGDEPTH); + + Path specPath = MergeOutputFormat.getMergeOutputPath(job); + Path tmpPath = Utilities.toTempPath(specPath); + Path taskTmpPath = Utilities.toTaskTempPath(specPath); + outWriter = null; + prevPath = null; + reader = null; + fdis = null; + updatePaths(tmpPath, taskTmpPath); + try { + fs = specPath.getFileSystem(job); + autoDelete = fs.deleteOnExit(outPath); + } catch (IOException e) { + this.exception = true; + throw new RuntimeException(e); + } + } + + private void updatePaths(Path tmpPath, Path taskTmpPath) { + String taskId = Utilities.getTaskId(jc); + this.tmpPath = tmpPath; + this.taskTmpPath = taskTmpPath; + finalPath = new Path(tmpPath, taskId); + outPath = new Path(taskTmpPath, Utilities.toTempPath(taskId)); + } + + @Override + public void map(Object key, OrcFileValueWrapper value, OutputCollector output, + Reporter reporter) throws IOException { + try { + + OrcFileKeyWrapper k = null; + if (key instanceof CombineHiveKey) { + k = (OrcFileKeyWrapper) ((CombineHiveKey) key).getKey(); + } else { + k = (OrcFileKeyWrapper) key; + } + + /** + * 1. boolean isListBucketingAlterTableConcatenate will be true only if it is alter table ... + * concatenate on stored-as-dir so it will handle list bucketing alter table merge in the if + * cause with the help of fixTmpPathConcatenate 2. If it is DML, + * isListBucketingAlterTableConcatenate will be false so that it will be handled by else + * cause. In this else cause, we have another if check. 2.1 the if check will make sure DP or + * LB, we will fix path with the help of fixTmpPath(..). Since both has sub-directories. it + * includes SP + LB. 2.2 only SP without LB, we dont fix path. + */ + + // Fix temp path for alter table ... concatenate + if (isListBucketingAlterTableConcatenate) { + if (this.tmpPathFixedConcatenate) { + checkPartitionsMatch(k.getInputPath().getParent()); + } else { + fixTmpPathConcatenate(k.getInputPath().getParent()); + tmpPathFixedConcatenate = true; + } + } else { + if (hasDynamicPartitions || (listBucketingDepth > 0)) { + if (tmpPathFixed) { + checkPartitionsMatch(k.getInputPath().getParent()); + } else { + // We haven't fixed the TMP path for this mapper yet + fixTmpPath(k.getInputPath().getParent()); + tmpPathFixed = true; + } + } + } + + if (prevPath == null) { + prevPath = k.getInputPath(); + reader = OrcFile.createReader(fs, k.inputPath); + } + + // store the orc configuration from the first file. All other files should + // match this configuration before merging + if (outWriter == null) { + compression = k.getCompression(); + compressBuffSize = k.getCompressBufferSize(); + version = k.getVersionList(); + columnCount = k.getTypes().get(0).getSubtypesCount(); + rowIndexStride = k.getRowIndexStride(); + + // block size and stripe size will be from config + outWriter = OrcFile.createWriter(outPath, OrcFile.writerOptions(jc) + .compress(compression) + .inspector(reader.getObjectInspector())); + } + + // check compatibility with subsequent files + if ((k.getTypes().get(0).getSubtypesCount() != columnCount)) { + throw new IOException("ORCFileMerge failed because the input files are not compatible." + + " Column counts does not match."); + } + + if (!k.compression.equals(compression)) { + throw new IOException("ORCFileMerge failed because the input files are not compatible." + + " Compression codec does not match."); + } + + if (k.compressBufferSize != compressBuffSize) { + throw new IOException("ORCFileMerge failed because the input files are not compatible." + + " Compression buffer size does not match."); + + } + + if (!k.versionList.equals(version)) { + throw new IOException("ORCFileMerge failed because the input files are not compatible." + + " Version does not match."); + } + + if (k.rowIndexStride != rowIndexStride) { + throw new IOException("ORCFileMerge failed because the input files are not compatible." + + " Row index stride does not match."); + } + + // next file in the path + if (!k.getInputPath().equals(prevPath)) { + reader = OrcFile.createReader(fs, k.inputPath); + } + + // initialize buffer to read the entire stripe + buffer = new byte[(int) value.stripeInformation.getLength()]; + fdis = fs.open(k.inputPath); + fdis.readFully(value.stripeInformation.getOffset(), buffer, 0, + (int) value.stripeInformation.getLength()); + + // append the stripe buffer to the new ORC file + ((WriterImpl) outWriter).appendStripe(buffer, value.getStripeInformation(), + value.getStripeStatistics()); + + LOG.info("Merged stripe from file " + k.inputPath + " [ offset : " + + value.getStripeInformation().getOffset() + " length: " + + value.getStripeInformation().getLength() + " ]"); + + // add user metadata to footer in case of any + if (value.isLastStripeInFile()) { + ((WriterImpl) outWriter).appendUserMetadata(value.getUserMetadata()); + } + } catch (Throwable e) { + this.exception = true; + close(); + throw new IOException(e); + } + } + + /** + * Validates that each input path belongs to the same partition since each mapper merges the input + * to a single output directory + * @param inputPath + * @throws HiveException + */ + private void checkPartitionsMatch(Path inputPath) throws HiveException { + if (!dpPath.equals(inputPath)) { + // Temp partition input path does not match exist temp path + String msg = "Multiple partitions for one block merge mapper: " + dpPath + " NOT EQUAL TO " + + inputPath; + LOG.error(msg); + throw new HiveException(msg); + } + } + + /** + * Fixes tmpPath to point to the correct partition. Before this is called, tmpPath will default to + * the root tmp table dir fixTmpPath(..) works for DP + LB + multiple skewed values + merge. + * reason: 1. fixTmpPath(..) compares inputPath and tmpDepth, find out path difference and put it + * into newPath. Then add newpath to existing this.tmpPath and this.taskTmpPath. 2. The path + * difference between inputPath and tmpDepth can be DP or DP+LB. It will automatically handle it. + * 3. For example, if inputpath is /-ext-10002/hr=a1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/ + * HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME tmppath is /_tmp.-ext-10000 newpath will be + * hr=a1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME Then, + * this.tmpPath and this.taskTmpPath will be update correctly. We have list_bucket_dml_6.q cover + * this case: DP + LP + multiple skewed values + merge. + * @param inputPath + * @throws HiveException + * @throws IOException + */ + private void fixTmpPath(Path inputPath) throws HiveException, IOException { + dpPath = inputPath; + Path newPath = new Path("."); + int inputDepth = inputPath.depth(); + int tmpDepth = tmpPath.depth(); + + // Build the path from bottom up + while (inputPath != null && inputDepth > tmpDepth) { + newPath = new Path(inputPath.getName(), newPath); + inputDepth--; + inputPath = inputPath.getParent(); + } + + Path newTmpPath = new Path(tmpPath, newPath); + Path newTaskTmpPath = new Path(taskTmpPath, newPath); + if (!fs.exists(newTmpPath)) { + fs.mkdirs(newTmpPath); + } + updatePaths(newTmpPath, newTaskTmpPath); + } + + /** + * Fixes tmpPath to point to the correct list bucketing sub-directories. Before this is called, + * tmpPath will default to the root tmp table dir Reason to add a new method instead of changing + * fixTmpPath() Reason 1: logic has slightly difference fixTmpPath(..) needs 2 variables in order + * to decide path delta which is in variable newPath. 1. inputPath.depth() 2. tmpPath.depth() + * fixTmpPathConcatenate needs 2 variables too but one of them is different from fixTmpPath(..) 1. + * inputPath.depth() 2. listBucketingDepth Reason 2: less risks The existing logic is a little not + * trivial around map() and fixTmpPath(). In order to ensure minimum impact on existing flow, we + * try to avoid change on existing code/flow but add new code for new feature. + * @param inputPath + * @throws HiveException + * @throws IOException + */ + private void fixTmpPathConcatenate(Path inputPath) throws HiveException, IOException { + dpPath = inputPath; + Path newPath = new Path("."); + + int depth = listBucketingDepth; + // Build the path from bottom up. pick up list bucketing subdirectories + while ((inputPath != null) && (depth > 0)) { + newPath = new Path(inputPath.getName(), newPath); + inputPath = inputPath.getParent(); + depth--; + } + + Path newTmpPath = new Path(tmpPath, newPath); + Path newTaskTmpPath = new Path(taskTmpPath, newPath); + if (!fs.exists(newTmpPath)) { + fs.mkdirs(newTmpPath); + } + updatePaths(newTmpPath, newTaskTmpPath); + } + + @Override + public void close() throws IOException { + // close writer + if (outWriter == null) { + return; + } + + if (fdis != null) { + fdis.close(); + fdis = null; + } + + outWriter.close(); + outWriter = null; + + if (!exception) { + FileStatus fss = fs.getFileStatus(outPath); + LOG.info("renamed path " + outPath + " to " + finalPath + " . File size is " + fss.getLen()); + if (!fs.rename(outPath, finalPath)) { + throw new IOException("Unable to rename output to " + finalPath); + } + } else { + if (!autoDelete) { + fs.delete(outPath, true); + } + } + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeInputFormat.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeInputFormat.java new file mode 100644 index 0000000..327402c --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeInputFormat.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +import java.io.IOException; +import org.apache.hadoop.mapred.FileInputFormat; +import org.apache.hadoop.mapred.FileSplit; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordReader; +import org.apache.hadoop.mapred.Reporter; + +public class OrcFileStripeMergeInputFormat extends + FileInputFormat { + + @Override + public RecordReader getRecordReader( + InputSplit split, JobConf job, Reporter reporter) throws IOException { + + reporter.setStatus(split.toString()); + + return new OrcFileStripeMergeRecordReader(job, (FileSplit) split); + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java new file mode 100644 index 0000000..db5bbb5 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapred.FileSplit; +import org.apache.hadoop.mapred.RecordReader; + +public class OrcFileStripeMergeRecordReader implements + RecordReader { + + private final Reader reader; + private final Path path; + protected Iterator iter; + protected List stripeStatistics; + private int stripeIdx; + private long start; + private long end; + + public OrcFileStripeMergeRecordReader(Configuration conf, FileSplit split) throws IOException { + path = split.getPath(); + start = split.getStart(); + end = start + split.getLength(); + FileSystem fs = path.getFileSystem(conf); + this.reader = OrcFile.createReader(path, OrcFile.readerOptions(conf).filesystem(fs)); + this.iter = reader.getStripes().iterator(); + this.stripeIdx = 0; + this.stripeStatistics = ((ReaderImpl) reader).getOrcProtoStripeStatistics(); + } + + public Class getKeyClass() { + return OrcFileKeyWrapper.class; + } + + public Class getValueClass() { + return OrcFileValueWrapper.class; + } + + public OrcFileKeyWrapper createKey() { + return new OrcFileKeyWrapper(); + } + + public OrcFileValueWrapper createValue() { + return new OrcFileValueWrapper(); + } + + @Override + public boolean next(OrcFileKeyWrapper key, OrcFileValueWrapper value) throws IOException { + return nextStripe(key, value); + } + + protected boolean nextStripe(OrcFileKeyWrapper keyWrapper, OrcFileValueWrapper valueWrapper) + throws IOException { + while (iter.hasNext()) { + StripeInformation si = iter.next(); + + // if stripe offset is outside the split boundary then ignore the current + // stripe as it will be handled by some other mapper. + if (si.getOffset() >= start && si.getOffset() < end) { + valueWrapper.setStripeStatistics(stripeStatistics.get(stripeIdx++)); + valueWrapper.setStripeInformation(si); + if (!iter.hasNext()) { + valueWrapper.setLastStripeInFile(true); + valueWrapper.setUserMetadata(((ReaderImpl) reader).getOrcProtoUserMetadata()); + } + keyWrapper.setInputPath(path); + keyWrapper.setCompression(reader.getCompression()); + keyWrapper.setCompressBufferSize(reader.getCompressionSize()); + keyWrapper.setVersionList(((ReaderImpl) reader).getFileMetaInfo().versionList); + keyWrapper.setRowIndexStride(reader.getRowIndexStride()); + keyWrapper.setTypes(reader.getTypes()); + } else { + continue; + } + return true; + } + + return false; + } + + /** + * Default progress will be based on number of files processed. + * @return 0.0 to 1.0 of the input byte range + */ + public float getProgress() throws IOException { + return 0.0f; + } + + public long getPos() throws IOException { + return 0; + } + + protected void seek(long pos) throws IOException { + } + + public long getStart() { + return 0; + } + + public void close() throws IOException { + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileValueWrapper.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileValueWrapper.java new file mode 100644 index 0000000..7389ef5 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileValueWrapper.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.io.WritableComparable; + +/** + * Value for OrcFileMergeMapper. Contains stripe related information for the + * current orc file that is being merged. + */ +public class OrcFileValueWrapper implements WritableComparable { + + protected StripeInformation stripeInformation; + protected OrcProto.StripeStatistics stripeStatistics; + protected List userMetadata; + protected boolean lastStripeInFile; + + public List getUserMetadata() { + return userMetadata; + } + + public void setUserMetadata(List userMetadata) { + this.userMetadata = userMetadata; + } + + public boolean isLastStripeInFile() { + return lastStripeInFile; + } + + public void setLastStripeInFile(boolean lastStripeInFile) { + this.lastStripeInFile = lastStripeInFile; + } + + public OrcProto.StripeStatistics getStripeStatistics() { + return stripeStatistics; + } + + public void setStripeStatistics(OrcProto.StripeStatistics stripeStatistics) { + this.stripeStatistics = stripeStatistics; + } + + public StripeInformation getStripeInformation() { + return stripeInformation; + } + + public void setStripeInformation(StripeInformation stripeInformation) { + this.stripeInformation = stripeInformation; + } + + @Override + public void write(DataOutput out) throws IOException { + throw new RuntimeException("Not supported."); + } + + @Override + public void readFields(DataInput in) throws IOException { + throw new RuntimeException("Not supported."); + } + + @Override + public int compareTo(OrcFileValueWrapper o) { + if (stripeInformation.getOffset() < o.getStripeInformation().getOffset()) { + return -1; + } else if (stripeInformation.getOffset() > o.getStripeInformation().getOffset()) { + return 1; + } else { + return 0; + } + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java index e3258e1..13f79df 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java @@ -34,6 +34,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.io.orc.OrcProto.Type; +import org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; import org.apache.hadoop.hive.ql.util.JavaDataModel; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -60,6 +61,7 @@ private final ObjectInspector inspector; private long deserializedSize = -1; private final Configuration conf; + private final List versionList; //serialized footer - Keeping this around for use by getFileMetaInfo() // will help avoid cpu cycles spend in deserializing at cost of increased @@ -306,6 +308,7 @@ static void checkOrcVersion(Log log, Path path, List version) { this.metadata = rInfo.metadata; this.footer = rInfo.footer; this.inspector = rInfo.inspector; + this.versionList = footerMetaData.versionList; } @@ -387,7 +390,8 @@ private static FileMetaInfo extractMetaInfoFromFooter(FileSystem fs, ps.getCompression().toString(), (int) ps.getCompressionBlockSize(), (int) ps.getMetadataLength(), - buffer + buffer, + ps.getVersionList() ); } @@ -446,18 +450,26 @@ private static FileMetaInfo extractMetaInfoFromFooter(FileSystem fs, final int bufferSize; final int metadataSize; final ByteBuffer footerBuffer; + final List versionList; + + FileMetaInfo(String compressionType, int bufferSize, int metadataSize, + ByteBuffer footerBuffer) { + this(compressionType, bufferSize, metadataSize, footerBuffer, null); + } + FileMetaInfo(String compressionType, int bufferSize, int metadataSize, - ByteBuffer footerBuffer){ + ByteBuffer footerBuffer, List versionList){ this.compressionType = compressionType; this.bufferSize = bufferSize; this.metadataSize = metadataSize; this.footerBuffer = footerBuffer; + this.versionList = versionList; } } public FileMetaInfo getFileMetaInfo(){ return new FileMetaInfo(compressionKind.toString(), bufferSize, - metadataSize, footerByteBuffer); + metadataSize, footerByteBuffer, versionList); } @@ -629,4 +641,11 @@ public Metadata getMetadata() throws IOException { return new Metadata(metadata); } + List getOrcProtoStripeStatistics() { + return metadata.getStripeStatsList(); + } + + public List getOrcProtoUserMetadata() { + return footer.getMetadataList(); + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java index 020b4cf..76b4d03 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java @@ -28,8 +28,6 @@ import java.util.Map; import java.util.TreeMap; -import com.google.common.annotations.VisibleForTesting; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -43,8 +41,8 @@ import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry; import org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics; import org.apache.hadoop.hive.ql.io.orc.OrcProto.Type; +import org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem; import org.apache.hadoop.hive.ql.util.JavaDataModel; -import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; @@ -73,6 +71,7 @@ import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Text; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.google.protobuf.ByteString; import com.google.protobuf.CodedOutputStream; @@ -2218,4 +2217,78 @@ public synchronized long writeIntermediateFooter() throws IOException { } return rawWriter.getPos(); } + + void appendStripe(byte[] stripe, StripeInformation stripeInfo, + OrcProto.StripeStatistics stripeStatistics) throws IOException { + appendStripe(stripe, 0, stripe.length, stripeInfo, stripeStatistics); + } + + void appendStripe(byte[] stripe, int offset, int length, + StripeInformation stripeInfo, + OrcProto.StripeStatistics stripeStatistics) throws IOException { + getStream(); + long start = rawWriter.getPos(); + + long stripeLen = length; + long availBlockSpace = blockSize - (start % blockSize); + + // see if stripe can fit in the current hdfs block, else pad the remaining + // space in the block + if (stripeLen < blockSize && stripeLen > availBlockSpace && + addBlockPadding) { + byte[] pad = new byte[(int) Math.min(HDFS_BUFFER_SIZE, availBlockSpace)]; + LOG.info(String.format("Padding ORC by %d bytes while merging..", + availBlockSpace)); + start += availBlockSpace; + while (availBlockSpace > 0) { + int writeLen = (int) Math.min(availBlockSpace, pad.length); + rawWriter.write(pad, 0, writeLen); + availBlockSpace -= writeLen; + } + } + + rawWriter.write(stripe); + rowsInStripe = stripeStatistics.getColStats(0).getNumberOfValues(); + rowCount += rowsInStripe; + + // since we have already written the stripe, just update stripe statistics + treeWriter.stripeStatsBuilders.add(stripeStatistics.toBuilder()); + + // update file level statistics + updateFileStatistics(stripeStatistics); + + // update stripe information + OrcProto.StripeInformation dirEntry = OrcProto.StripeInformation + .newBuilder() + .setOffset(start) + .setNumberOfRows(rowsInStripe) + .setIndexLength(stripeInfo.getIndexLength()) + .setDataLength(stripeInfo.getDataLength()) + .setFooterLength(stripeInfo.getFooterLength()) + .build(); + stripes.add(dirEntry); + + // reset it after writing the stripe + rowsInStripe = 0; + } + + private void updateFileStatistics(OrcProto.StripeStatistics stripeStatistics) { + List cs = stripeStatistics.getColStatsList(); + + // root element + treeWriter.fileStatistics.merge(ColumnStatisticsImpl.deserialize(cs.get(0))); + TreeWriter[] childWriters = treeWriter.getChildrenWriters(); + for (int i = 0; i < childWriters.length; i++) { + childWriters[i].fileStatistics.merge( + ColumnStatisticsImpl.deserialize(cs.get(i + 1))); + } + } + + void appendUserMetadata(List userMetadata) { + if (userMetadata != null) { + for (UserMetadataItem item : userMetadata) { + this.userMetadata.put(item.getName(), item.getValue()); + } + } + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java deleted file mode 100644 index 6ebda9a..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java +++ /dev/null @@ -1,376 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.io.rcfile.merge; - -import java.io.IOException; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Enumeration; -import java.util.List; - -import org.apache.commons.lang.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.Context; -import org.apache.hadoop.hive.ql.DriverContext; -import org.apache.hadoop.hive.ql.QueryPlan; -import org.apache.hadoop.hive.ql.exec.Task; -import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.exec.mr.HadoopJobExecHelper; -import org.apache.hadoop.hive.ql.exec.mr.HadoopJobExecHook; -import org.apache.hadoop.hive.ql.exec.mr.Throttle; -import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat; -import org.apache.hadoop.hive.ql.io.HiveOutputFormatImpl; -import org.apache.hadoop.hive.ql.plan.api.StageType; -import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; -import org.apache.hadoop.hive.shims.ShimLoader; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.mapred.Counters; -import org.apache.hadoop.mapred.FileInputFormat; -import org.apache.hadoop.mapred.InputFormat; -import org.apache.hadoop.mapred.JobClient; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.RunningJob; -import org.apache.log4j.Appender; -import org.apache.log4j.FileAppender; -import org.apache.log4j.LogManager; - -@SuppressWarnings( { "deprecation", "unchecked" }) -public class BlockMergeTask extends Task implements Serializable, - HadoopJobExecHook { - - private static final long serialVersionUID = 1L; - - protected transient JobConf job; - protected HadoopJobExecHelper jobExecHelper; - - @Override - public void initialize(HiveConf conf, QueryPlan queryPlan, - DriverContext driverContext) { - super.initialize(conf, queryPlan, driverContext); - job = new JobConf(conf, BlockMergeTask.class); - jobExecHelper = new HadoopJobExecHelper(job, this.console, this, this); - } - - @Override - public boolean requireLock() { - return true; - } - - boolean success = true; - - @Override - /** - * start a new map-reduce job to do the merge, almost the same as ExecDriver. - */ - public int execute(DriverContext driverContext) { - HiveConf.setVar(job, HiveConf.ConfVars.HIVEINPUTFORMAT, - CombineHiveInputFormat.class.getName()); - success = true; - ShimLoader.getHadoopShims().prepareJobOutput(job); - job.setOutputFormat(HiveOutputFormatImpl.class); - job.setMapperClass(work.getMapperClass()); - - Context ctx = driverContext.getCtx(); - boolean ctxCreated = false; - try { - if (ctx == null) { - ctx = new Context(job); - ctxCreated = true; - } - }catch (IOException e) { - e.printStackTrace(); - console.printError("Error launching map-reduce job", "\n" - + org.apache.hadoop.util.StringUtils.stringifyException(e)); - return 5; - } - - job.setMapOutputKeyClass(NullWritable.class); - job.setMapOutputValueClass(NullWritable.class); - if(work.getNumMapTasks() != null) { - job.setNumMapTasks(work.getNumMapTasks()); - } - - // zero reducers - job.setNumReduceTasks(0); - - if (work.getMinSplitSize() != null) { - HiveConf.setLongVar(job, HiveConf.ConfVars.MAPREDMINSPLITSIZE, work - .getMinSplitSize().longValue()); - } - - if (work.getInputformat() != null) { - HiveConf.setVar(job, HiveConf.ConfVars.HIVEINPUTFORMAT, work - .getInputformat()); - } - - String inpFormat = HiveConf.getVar(job, HiveConf.ConfVars.HIVEINPUTFORMAT); - if ((inpFormat == null) || (!StringUtils.isNotBlank(inpFormat))) { - inpFormat = ShimLoader.getHadoopShims().getInputFormatClassName(); - } - - LOG.info("Using " + inpFormat); - - try { - job.setInputFormat((Class) (Class - .forName(inpFormat))); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e.getMessage()); - } - - Path outputPath = this.work.getOutputDir(); - Path tempOutPath = Utilities.toTempPath(outputPath); - try { - FileSystem fs = tempOutPath.getFileSystem(job); - if (!fs.exists(tempOutPath)) { - fs.mkdirs(tempOutPath); - } - } catch (IOException e) { - console.printError("Can't make path " + outputPath + " : " + e.getMessage()); - return 6; - } - - RCFileBlockMergeOutputFormat.setMergeOutputPath(job, outputPath); - - job.setOutputKeyClass(NullWritable.class); - job.setOutputValueClass(NullWritable.class); - - HiveConf.setBoolVar(job, - HiveConf.ConfVars.HIVEMERGECURRENTJOBHASDYNAMICPARTITIONS, - work.hasDynamicPartitions()); - - HiveConf.setBoolVar(job, - HiveConf.ConfVars.HIVEMERGECURRENTJOBCONCATENATELISTBUCKETING, - work.isListBucketingAlterTableConcatenate()); - - HiveConf.setIntVar( - job, - HiveConf.ConfVars.HIVEMERGECURRENTJOBCONCATENATELISTBUCKETINGDEPTH, - ((work.getListBucketingCtx() == null) ? 0 : work.getListBucketingCtx() - .calculateListBucketingLevel())); - - int returnVal = 0; - RunningJob rj = null; - boolean noName = StringUtils.isEmpty(HiveConf.getVar(job, - HiveConf.ConfVars.HADOOPJOBNAME)); - - String jobName = null; - if (noName && this.getQueryPlan() != null) { - int maxlen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH); - jobName = Utilities.abbreviate(this.getQueryPlan().getQueryStr(), - maxlen - 6); - } - - if (noName) { - // This is for a special case to ensure unit tests pass - HiveConf.setVar(job, HiveConf.ConfVars.HADOOPJOBNAME, - jobName != null ? jobName : "JOB" + Utilities.randGen.nextInt()); - } - - try { - addInputPaths(job, work); - - Utilities.setMapWork(job, work, ctx.getMRTmpPath(), true); - - // remove the pwd from conf file so that job tracker doesn't show this - // logs - String pwd = HiveConf.getVar(job, HiveConf.ConfVars.METASTOREPWD); - if (pwd != null) { - HiveConf.setVar(job, HiveConf.ConfVars.METASTOREPWD, "HIVE"); - } - JobClient jc = new JobClient(job); - - String addedJars = Utilities.getResourceFiles(job, SessionState.ResourceType.JAR); - if (!addedJars.isEmpty()) { - job.set("tmpjars", addedJars); - } - - // make this client wait if job trcker is not behaving well. - Throttle.checkJobTracker(job, LOG); - - // Finally SUBMIT the JOB! - rj = jc.submitJob(job); - - returnVal = jobExecHelper.progress(rj, jc, null); - success = (returnVal == 0); - - } catch (Exception e) { - e.printStackTrace(); - String mesg = " with exception '" + Utilities.getNameMessage(e) + "'"; - if (rj != null) { - mesg = "Ended Job = " + rj.getJobID() + mesg; - } else { - mesg = "Job Submission failed" + mesg; - } - - // Has to use full name to make sure it does not conflict with - // org.apache.commons.lang.StringUtils - console.printError(mesg, "\n" - + org.apache.hadoop.util.StringUtils.stringifyException(e)); - - success = false; - returnVal = 1; - } finally { - try { - if (ctxCreated) { - ctx.clear(); - } - if (rj != null) { - if (returnVal != 0) { - rj.killJob(); - } - HadoopJobExecHelper.runningJobs.remove(rj); - jobID = rj.getID().toString(); - } - RCFileMergeMapper.jobClose(outputPath, success, job, console, - work.getDynPartCtx(), null); - } catch (Exception e) { - } - } - - return (returnVal); - } - - private void addInputPaths(JobConf job, MergeWork work) { - for (Path path : work.getInputPaths()) { - FileInputFormat.addInputPath(job, path); - } - } - - @Override - public String getName() { - return "RCFile Merge"; - } - - public static String INPUT_SEPERATOR = ":"; - - public static void main(String[] args) { - String inputPathStr = null; - String outputDir = null; - String jobConfFileName = null; - - try { - for (int i = 0; i < args.length; i++) { - if (args[i].equals("-input")) { - inputPathStr = args[++i]; - } else if (args[i].equals("-jobconffile")) { - jobConfFileName = args[++i]; - } else if (args[i].equals("-outputDir")) { - outputDir = args[++i]; - } - } - } catch (IndexOutOfBoundsException e) { - System.err.println("Missing argument to option"); - printUsage(); - } - - if (inputPathStr == null || outputDir == null - || outputDir.trim().equals("")) { - printUsage(); - } - - List inputPaths = new ArrayList(); - String[] paths = inputPathStr.split(INPUT_SEPERATOR); - if (paths == null || paths.length == 0) { - printUsage(); - } - - FileSystem fs = null; - JobConf conf = new JobConf(BlockMergeTask.class); - for (String path : paths) { - try { - Path pathObj = new Path(path); - if (fs == null) { - fs = FileSystem.get(pathObj.toUri(), conf); - } - FileStatus fstatus = fs.getFileStatus(pathObj); - if (fstatus.isDir()) { - FileStatus[] fileStatus = fs.listStatus(pathObj); - for (FileStatus st : fileStatus) { - inputPaths.add(st.getPath()); - } - } else { - inputPaths.add(fstatus.getPath()); - } - } catch (IOException e) { - e.printStackTrace(System.err); - } - } - - if (jobConfFileName != null) { - conf.addResource(new Path(jobConfFileName)); - } - HiveConf hiveConf = new HiveConf(conf, BlockMergeTask.class); - - Log LOG = LogFactory.getLog(BlockMergeTask.class.getName()); - boolean isSilent = HiveConf.getBoolVar(conf, - HiveConf.ConfVars.HIVESESSIONSILENT); - LogHelper console = new LogHelper(LOG, isSilent); - - // print out the location of the log file for the user so - // that it's easy to find reason for local mode execution failures - for (Appender appender : Collections - .list((Enumeration) LogManager.getRootLogger() - .getAllAppenders())) { - if (appender instanceof FileAppender) { - console.printInfo("Execution log at: " - + ((FileAppender) appender).getFile()); - } - } - - MergeWork mergeWork = new MergeWork(inputPaths, new Path(outputDir)); - DriverContext driverCxt = new DriverContext(); - BlockMergeTask taskExec = new BlockMergeTask(); - taskExec.initialize(hiveConf, null, driverCxt); - taskExec.setWork(mergeWork); - int ret = taskExec.execute(driverCxt); - - if (ret != 0) { - System.exit(2); - } - - } - - private static void printUsage() { - System.err.println("BlockMergeTask -input " - + "-outputDir outputDir [-jobconffile ] "); - System.exit(1); - } - - @Override - public StageType getType() { - return StageType.MAPRED; - } - - @Override - public boolean checkFatalErrors(Counters ctrs, StringBuilder errMsg) { - return false; - } - - @Override - public void logPlanProgress(SessionState ss) throws IOException { - // no op - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/MergeOutputFormat.java ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/MergeOutputFormat.java new file mode 100644 index 0000000..bdd6fa9 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/MergeOutputFormat.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.rcfile.merge; + +import java.io.IOException; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapred.FileOutputFormat; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordWriter; +import org.apache.hadoop.util.Progressable; + +public class MergeOutputFormat extends + FileOutputFormat { + + public static void setMergeOutputPath(JobConf job, Path path) { + job.set("hive.merge.output.dir", path.toString()); + } + + public static Path getMergeOutputPath(JobConf conf) { + String name = conf.get("hive.merge.output.dir"); + return name == null ? null: new Path(name); + } + + @Override + public RecordWriter getRecordWriter( + FileSystem ignored, JobConf job, String name, Progressable progress) + throws IOException { + throw new RuntimeException("Not implemented."); + } +} \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/MergeTask.java ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/MergeTask.java new file mode 100644 index 0000000..0dc83da --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/MergeTask.java @@ -0,0 +1,415 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.rcfile.merge; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Enumeration; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.Context; +import org.apache.hadoop.hive.ql.DriverContext; +import org.apache.hadoop.hive.ql.QueryPlan; +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.exec.mr.HadoopJobExecHelper; +import org.apache.hadoop.hive.ql.exec.mr.HadoopJobExecHook; +import org.apache.hadoop.hive.ql.exec.mr.Throttle; +import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat; +import org.apache.hadoop.hive.ql.io.HiveOutputFormatImpl; +import org.apache.hadoop.hive.ql.io.RCFileInputFormat; +import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx; +import org.apache.hadoop.hive.ql.plan.api.StageType; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; +import org.apache.hadoop.hive.shims.ShimLoader; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapred.Counters; +import org.apache.hadoop.mapred.FileInputFormat; +import org.apache.hadoop.mapred.InputFormat; +import org.apache.hadoop.mapred.JobClient; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.RunningJob; +import org.apache.log4j.Appender; +import org.apache.log4j.FileAppender; +import org.apache.log4j.LogManager; + +@SuppressWarnings( { "deprecation", "unchecked" }) +public class MergeTask extends Task implements Serializable, + HadoopJobExecHook { + + private static final long serialVersionUID = 1L; + + public static String BACKUP_PREFIX = "_backup."; + + protected transient JobConf job; + protected HadoopJobExecHelper jobExecHelper; + + @Override + public void initialize(HiveConf conf, QueryPlan queryPlan, + DriverContext driverContext) { + super.initialize(conf, queryPlan, driverContext); + job = new JobConf(conf, MergeTask.class); + jobExecHelper = new HadoopJobExecHelper(job, this.console, this, this); + } + + @Override + public boolean requireLock() { + return true; + } + + boolean success = true; + + @Override + /** + * start a new map-reduce job to do the merge, almost the same as ExecDriver. + */ + public int execute(DriverContext driverContext) { + HiveConf.setVar(job, HiveConf.ConfVars.HIVEINPUTFORMAT, + CombineHiveInputFormat.class.getName()); + success = true; + ShimLoader.getHadoopShims().prepareJobOutput(job); + job.setOutputFormat(HiveOutputFormatImpl.class); + Class mapperClass = work.getMapperClass(work.getSourceTableInputFormat()); + LOG.info("Using " + mapperClass.getCanonicalName() + " mapper class."); + job.setMapperClass(mapperClass); + + Context ctx = driverContext.getCtx(); + boolean ctxCreated = false; + try { + if (ctx == null) { + ctx = new Context(job); + ctxCreated = true; + } + }catch (IOException e) { + e.printStackTrace(); + console.printError("Error launching map-reduce job", "\n" + + org.apache.hadoop.util.StringUtils.stringifyException(e)); + return 5; + } + + job.setMapOutputKeyClass(NullWritable.class); + job.setMapOutputValueClass(NullWritable.class); + if(work.getNumMapTasks() != null) { + job.setNumMapTasks(work.getNumMapTasks()); + } + + // zero reducers + job.setNumReduceTasks(0); + + if (work.getMinSplitSize() != null) { + HiveConf.setLongVar(job, HiveConf.ConfVars.MAPREDMINSPLITSIZE, work + .getMinSplitSize().longValue()); + } + + if (work.getInputformat() != null) { + HiveConf.setVar(job, HiveConf.ConfVars.HIVEINPUTFORMAT, work + .getInputformat()); + } + + String inpFormat = HiveConf.getVar(job, HiveConf.ConfVars.HIVEINPUTFORMAT); + if ((inpFormat == null) || (!StringUtils.isNotBlank(inpFormat))) { + inpFormat = ShimLoader.getHadoopShims().getInputFormatClassName(); + } + + LOG.info("Using " + inpFormat); + + try { + job.setInputFormat((Class) (Class + .forName(inpFormat))); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e.getMessage()); + } + + Path outputPath = this.work.getOutputDir(); + Path tempOutPath = Utilities.toTempPath(outputPath); + try { + FileSystem fs = tempOutPath.getFileSystem(job); + if (!fs.exists(tempOutPath)) { + fs.mkdirs(tempOutPath); + } + } catch (IOException e) { + console.printError("Can't make path " + outputPath + " : " + e.getMessage()); + return 6; + } + + MergeOutputFormat.setMergeOutputPath(job, outputPath); + + job.setOutputKeyClass(NullWritable.class); + job.setOutputValueClass(NullWritable.class); + + HiveConf.setBoolVar(job, + HiveConf.ConfVars.HIVEMERGECURRENTJOBHASDYNAMICPARTITIONS, + work.hasDynamicPartitions()); + + HiveConf.setBoolVar(job, + HiveConf.ConfVars.HIVEMERGECURRENTJOBCONCATENATELISTBUCKETING, + work.isListBucketingAlterTableConcatenate()); + + HiveConf.setIntVar( + job, + HiveConf.ConfVars.HIVEMERGECURRENTJOBCONCATENATELISTBUCKETINGDEPTH, + ((work.getListBucketingCtx() == null) ? 0 : work.getListBucketingCtx() + .calculateListBucketingLevel())); + + int returnVal = 0; + RunningJob rj = null; + boolean noName = StringUtils.isEmpty(HiveConf.getVar(job, + HiveConf.ConfVars.HADOOPJOBNAME)); + + String jobName = null; + if (noName && this.getQueryPlan() != null) { + int maxlen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH); + jobName = Utilities.abbreviate(this.getQueryPlan().getQueryStr(), + maxlen - 6); + } + + if (noName) { + // This is for a special case to ensure unit tests pass + HiveConf.setVar(job, HiveConf.ConfVars.HADOOPJOBNAME, + jobName != null ? jobName : "JOB" + Utilities.randGen.nextInt()); + } + + try { + addInputPaths(job, work); + + Utilities.setMapWork(job, work, ctx.getMRTmpPath(), true); + + // remove the pwd from conf file so that job tracker doesn't show this + // logs + String pwd = HiveConf.getVar(job, HiveConf.ConfVars.METASTOREPWD); + if (pwd != null) { + HiveConf.setVar(job, HiveConf.ConfVars.METASTOREPWD, "HIVE"); + } + JobClient jc = new JobClient(job); + + String addedJars = Utilities.getResourceFiles(job, SessionState.ResourceType.JAR); + if (!addedJars.isEmpty()) { + job.set("tmpjars", addedJars); + } + + // make this client wait if job trcker is not behaving well. + Throttle.checkJobTracker(job, LOG); + + // Finally SUBMIT the JOB! + rj = jc.submitJob(job); + + returnVal = jobExecHelper.progress(rj, jc, null); + success = (returnVal == 0); + + } catch (Exception e) { + e.printStackTrace(); + String mesg = " with exception '" + Utilities.getNameMessage(e) + "'"; + if (rj != null) { + mesg = "Ended Job = " + rj.getJobID() + mesg; + } else { + mesg = "Job Submission failed" + mesg; + } + + // Has to use full name to make sure it does not conflict with + // org.apache.commons.lang.StringUtils + console.printError(mesg, "\n" + + org.apache.hadoop.util.StringUtils.stringifyException(e)); + + success = false; + returnVal = 1; + } finally { + try { + if (ctxCreated) { + ctx.clear(); + } + if (rj != null) { + if (returnVal != 0) { + rj.killJob(); + } + HadoopJobExecHelper.runningJobs.remove(rj); + jobID = rj.getID().toString(); + } + closeJob(outputPath, success, job, console, work.getDynPartCtx(), null); + } catch (Exception e) { + } + } + + return (returnVal); + } + + private Path backupOutputPath(FileSystem fs, Path outpath, JobConf job) + throws IOException, HiveException { + if (fs.exists(outpath)) { + Path backupPath = new Path(outpath.getParent(), BACKUP_PREFIX + + outpath.getName()); + Utilities.rename(fs, outpath, backupPath); + return backupPath; + } else { + return null; + } + } + + private void closeJob(Path outputPath, boolean success, JobConf job, + LogHelper console, DynamicPartitionCtx dynPartCtx, Reporter reporter + ) throws HiveException, IOException { + FileSystem fs = outputPath.getFileSystem(job); + Path backupPath = backupOutputPath(fs, outputPath, job); + Utilities.mvFileToFinalPath(outputPath, job, success, LOG, dynPartCtx, null, + reporter); + fs.delete(backupPath, true); + } + + private void addInputPaths(JobConf job, MergeWork work) { + for (Path path : work.getInputPaths()) { + FileInputFormat.addInputPath(job, path); + } + } + + @Override + public String getName() { + return "MergeTask"; + } + + public static String INPUT_SEPERATOR = ":"; + + public static void main(String[] args) { + String inputPathStr = null; + String outputDir = null; + String jobConfFileName = null; + String format = null; + + try { + for (int i = 0; i < args.length; i++) { + if (args[i].equals("-input")) { + inputPathStr = args[++i]; + } else if (args[i].equals("-jobconffile")) { + jobConfFileName = args[++i]; + } else if (args[i].equals("-outputDir")) { + outputDir = args[++i]; + } else if (args[i].equals("-format")) { + format = args[++i]; + } + } + } catch (IndexOutOfBoundsException e) { + System.err.println("Missing argument to option"); + printUsage(); + } + + if (inputPathStr == null || outputDir == null + || outputDir.trim().equals("")) { + printUsage(); + } + + List inputPaths = new ArrayList(); + String[] paths = inputPathStr.split(INPUT_SEPERATOR); + if (paths == null || paths.length == 0) { + printUsage(); + } + + FileSystem fs = null; + JobConf conf = new JobConf(MergeTask.class); + for (String path : paths) { + try { + Path pathObj = new Path(path); + if (fs == null) { + fs = FileSystem.get(pathObj.toUri(), conf); + } + FileStatus fstatus = fs.getFileStatus(pathObj); + if (fstatus.isDir()) { + FileStatus[] fileStatus = fs.listStatus(pathObj); + for (FileStatus st : fileStatus) { + inputPaths.add(st.getPath()); + } + } else { + inputPaths.add(fstatus.getPath()); + } + } catch (IOException e) { + e.printStackTrace(System.err); + } + } + + if (jobConfFileName != null) { + conf.addResource(new Path(jobConfFileName)); + } + HiveConf hiveConf = new HiveConf(conf, MergeTask.class); + + Log LOG = LogFactory.getLog(MergeTask.class.getName()); + boolean isSilent = HiveConf.getBoolVar(conf, + HiveConf.ConfVars.HIVESESSIONSILENT); + LogHelper console = new LogHelper(LOG, isSilent); + + // print out the location of the log file for the user so + // that it's easy to find reason for local mode execution failures + for (Appender appender : Collections + .list((Enumeration) LogManager.getRootLogger() + .getAllAppenders())) { + if (appender instanceof FileAppender) { + console.printInfo("Execution log at: " + + ((FileAppender) appender).getFile()); + } + } + + MergeWork mergeWork = null; + if (format.equals("rcfile")) { + mergeWork = new MergeWork(inputPaths, new Path(outputDir), RCFileInputFormat.class); + } else if (format.equals("orcfile")) { + mergeWork = new MergeWork(inputPaths, new Path(outputDir), OrcInputFormat.class); + } + + DriverContext driverCxt = new DriverContext(); + MergeTask taskExec = new MergeTask(); + taskExec.initialize(hiveConf, null, driverCxt); + taskExec.setWork(mergeWork); + int ret = taskExec.execute(driverCxt); + + if (ret != 0) { + System.exit(2); + } + + } + + private static void printUsage() { + System.err.println("MergeTask -format -input " + + "-outputDir outputDir [-jobconffile ] "); + System.exit(1); + } + + @Override + public StageType getType() { + return StageType.MAPRED; + } + + @Override + public boolean checkFatalErrors(Counters ctrs, StringBuilder errMsg) { + return false; + } + + @Override + public void logPlanProgress(SessionState ss) throws IOException { + // no op + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/MergeWork.java ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/MergeWork.java index edb7b60..d72846a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/MergeWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/MergeWork.java @@ -30,6 +30,10 @@ import org.apache.hadoop.hive.common.HiveStatsUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat; +import org.apache.hadoop.hive.ql.io.RCFileInputFormat; +import org.apache.hadoop.hive.ql.io.orc.OrcFileMergeMapper; +import org.apache.hadoop.hive.ql.io.orc.OrcFileStripeMergeInputFormat; +import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.ListBucketingCtx; @@ -39,7 +43,7 @@ import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.Mapper; -@Explain(displayName = "Block level merge") +@Explain(displayName = "Merge Work") public class MergeWork extends MapWork implements Serializable { private static final long serialVersionUID = 1L; @@ -50,23 +54,31 @@ private DynamicPartitionCtx dynPartCtx; private boolean isListBucketingAlterTableConcatenate; private ListBucketingCtx listBucketingCtx; + private Class srcTblInputFormat; public MergeWork() { } - public MergeWork(List inputPaths, Path outputDir) { - this(inputPaths, outputDir, false, null); + public MergeWork(List inputPaths, Path outputDir, + Class srcTblInputFormat) { + this(inputPaths, outputDir, false, null, srcTblInputFormat); } public MergeWork(List inputPaths, Path outputDir, - boolean hasDynamicPartitions, DynamicPartitionCtx dynPartCtx) { + boolean hasDynamicPartitions, DynamicPartitionCtx dynPartCtx, + Class srcTblInputFormat) { super(); this.inputPaths = inputPaths; this.outputDir = outputDir; this.hasDynamicPartitions = hasDynamicPartitions; this.dynPartCtx = dynPartCtx; + this.srcTblInputFormat = srcTblInputFormat; PartitionDesc partDesc = new PartitionDesc(); - partDesc.setInputFileFormatClass(RCFileBlockMergeInputFormat.class); + if(srcTblInputFormat.equals(OrcInputFormat.class)) { + partDesc.setInputFileFormatClass(OrcFileStripeMergeInputFormat.class); + } else if(srcTblInputFormat.equals(RCFileInputFormat.class)) { + partDesc.setInputFileFormatClass(RCFileBlockMergeInputFormat.class); + } if(this.getPathToPartitionInfo() == null) { this.setPathToPartitionInfo(new LinkedHashMap()); } @@ -91,8 +103,13 @@ public void setOutputDir(Path outputDir) { this.outputDir = outputDir; } - public Class getMapperClass() { - return RCFileMergeMapper.class; + public Class getMapperClass(Class klass) { + if (klass.equals(RCFileInputFormat.class)) { + return RCFileMergeMapper.class; + } else if (klass.equals(OrcInputFormat.class)) { + return OrcFileMergeMapper.class; + } + return null; } @Override @@ -122,7 +139,13 @@ public void setHasDynamicPartitions(boolean hasDynamicPartitions) { public void resolveDynamicPartitionStoredAsSubDirsMerge(HiveConf conf, Path path, TableDesc tblDesc, ArrayList aliases, PartitionDesc partDesc) { - String inputFormatClass = conf.getVar(HiveConf.ConfVars.HIVEMERGEINPUTFORMATBLOCKLEVEL); + String inputFormatClass = null; + if (tblDesc.getInputFileFormatClass().equals(RCFileInputFormat.class)) { + inputFormatClass = conf.getVar(HiveConf.ConfVars.HIVEMERGEINPUTFORMATBLOCKLEVEL); + } else if (tblDesc.getInputFileFormatClass().equals(OrcInputFormat.class)){ + inputFormatClass = conf.getVar(HiveConf.ConfVars.HIVEMERGEINPUTFORMATSTRIPELEVEL); + } + try { partDesc.setInputFileFormatClass((Class ) Class.forName(inputFormatClass)); @@ -208,4 +231,29 @@ public boolean isListBucketingAlterTableConcatenate() { return isListBucketingAlterTableConcatenate; } + public Class getSourceTableInputFormat() { + return srcTblInputFormat; + } + + @Explain(displayName = "input format") + public String getStringifiedInputFormat() { + return srcTblInputFormat.getCanonicalName(); + } + + @Explain(displayName = "merge level") + public String getMergeLevel() { + if (srcTblInputFormat != null) { + if (srcTblInputFormat.equals(OrcInputFormat.class)) { + return "stripe"; + } else if (srcTblInputFormat.equals(RCFileInputFormat.class)) { + return "block"; + } + } + return null; + } + + public void setSourceTableInputFormat(Class srcTblInputFormat) { + this.srcTblInputFormat = srcTblInputFormat; + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileBlockMergeOutputFormat.java ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileBlockMergeOutputFormat.java deleted file mode 100644 index 180abc7..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileBlockMergeOutputFormat.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.io.rcfile.merge; - -import java.io.IOException; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.FileOutputFormat; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.RecordWriter; -import org.apache.hadoop.util.Progressable; - -public class RCFileBlockMergeOutputFormat extends - FileOutputFormat { - - public static void setMergeOutputPath(JobConf job, Path path) { - job.set("hive.rcfile.merge.output.dir", path.toString()); - } - - public static Path getMergeOutputPath(JobConf conf) { - String name = conf.get("hive.rcfile.merge.output.dir"); - return name == null ? null: new Path(name); - } - - @Override - public RecordWriter getRecordWriter( - FileSystem ignored, JobConf job, String name, Progressable progress) - throws IOException { - throw new RuntimeException("Not implemented."); - } -} \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileMergeMapper.java ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileMergeMapper.java index 066afa3..ce38b7f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileMergeMapper.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileMergeMapper.java @@ -84,7 +84,7 @@ public void configure(JobConf job) { listBucketingDepth = HiveConf.getIntVar(job, HiveConf.ConfVars.HIVEMERGECURRENTJOBCONCATENATELISTBUCKETINGDEPTH); - Path specPath = RCFileBlockMergeOutputFormat.getMergeOutputPath(job); + Path specPath = MergeOutputFormat.getMergeOutputPath(job); Path tmpPath = Utilities.toTempPath(specPath); Path taskTmpPath = Utilities.toTaskTempPath(specPath); updatePaths(tmpPath, taskTmpPath); @@ -296,28 +296,4 @@ public void close() throws IOException { } } - public static String BACKUP_PREFIX = "_backup."; - - public static Path backupOutputPath(FileSystem fs, Path outpath, JobConf job) - throws IOException, HiveException { - if (fs.exists(outpath)) { - Path backupPath = new Path(outpath.getParent(), BACKUP_PREFIX - + outpath.getName()); - Utilities.rename(fs, outpath, backupPath); - return backupPath; - } else { - return null; - } - } - - public static void jobClose(Path outputPath, boolean success, JobConf job, - LogHelper console, DynamicPartitionCtx dynPartCtx, Reporter reporter - ) throws HiveException, IOException { - FileSystem fs = outputPath.getFileSystem(job); - Path backupPath = backupOutputPath(fs, outputPath, job); - Utilities.mvFileToFinalPath(outputPath, job, success, LOG, dynPartCtx, null, - reporter); - fs.delete(backupPath, true); - } - } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java index f8b684f..e5b9c2b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java @@ -98,7 +98,8 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx opProcCtx, if (chDir) { // Merge the files in the destination table/partitions by creating Map-only merge job - // If underlying data is RCFile a RCFileBlockMerge task would be created. + // If underlying data is RCFile or OrcFile, RCFileBlockMerge task or + // OrcFileStripeMerge task would be created. LOG.info("using CombineHiveInputformat for the merge job"); GenMapRedUtils.createMRWorkForMergingFiles(fsOp, finalName, ctx.getDependencyTaskForMultiInsert(), ctx.getMvTask(), diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index 77f56c1..31ca3af 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -31,15 +31,16 @@ import java.util.Map.Entry; import java.util.Properties; import java.util.Set; -import java.lang.StringBuffer; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.ql.Context; +import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.ConditionalTask; import org.apache.hadoop.hive.ql.exec.DemuxOperator; @@ -64,11 +65,10 @@ import org.apache.hadoop.hive.ql.exec.mr.MapRedTask; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.io.RCFileInputFormat; +import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; import org.apache.hadoop.hive.ql.io.rcfile.merge.MergeWork; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMRUnionCtx; import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx; import org.apache.hadoop.hive.ql.optimizer.listbucketingpruner.ListBucketingPruner; @@ -1245,23 +1245,33 @@ public static void createMRWorkForMergingFiles (FileSinkOperator fsInput, MapWork cplan; Serializable work; - if (conf.getBoolVar(ConfVars.HIVEMERGERCFILEBLOCKLEVEL) && - fsInputDesc.getTableInfo().getInputFileFormatClass().equals(RCFileInputFormat.class)) { + if ((conf.getBoolVar(ConfVars.HIVEMERGERCFILEBLOCKLEVEL) && + fsInputDesc.getTableInfo().getInputFileFormatClass().equals(RCFileInputFormat.class)) || + (conf.getBoolVar(ConfVars.HIVEMERGEORCFILESTRIPELEVEL) && + fsInputDesc.getTableInfo().getInputFileFormatClass().equals(OrcInputFormat.class))) { // Check if InputFormatClass is valid - String inputFormatClass = conf.getVar(ConfVars.HIVEMERGEINPUTFORMATBLOCKLEVEL); + final String inputFormatClass; + if (fsInputDesc.getTableInfo().getInputFileFormatClass().equals(RCFileInputFormat.class)) { + inputFormatClass = conf.getVar(ConfVars.HIVEMERGEINPUTFORMATBLOCKLEVEL); + } else { + inputFormatClass = conf.getVar(ConfVars.HIVEMERGEINPUTFORMATSTRIPELEVEL); + } try { Class c = Class.forName(inputFormatClass); - LOG.info("RCFile format- Using block level merge"); - cplan = GenMapRedUtils.createRCFileMergeTask(fsInputDesc, finalName, + if(fsInputDesc.getTableInfo().getInputFileFormatClass().equals(OrcInputFormat.class)) { + LOG.info("OrcFile format - Using stripe level merge"); + } else { + LOG.info("RCFile format- Using block level merge"); + } + cplan = GenMapRedUtils.createMergeTask(fsInputDesc, finalName, dpCtx != null && dpCtx.getNumDPCols() > 0); work = cplan; } catch (ClassNotFoundException e) { String msg = "Illegal input format class: " + inputFormatClass; throw new SemanticException(msg); } - } else { cplan = createMRWorkForMergingFiles(conf, tsMerge, fsInputDesc); if (conf.getVar(ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) { @@ -1474,20 +1484,22 @@ private static MapWork createMRWorkForMergingFiles (HiveConf conf, } /** - * Create a block level merge task for RCFiles. + * Create a block level merge task for RCFiles or stripe level merge task for + * ORCFiles * * @param fsInputDesc * @param finalName - * @return MergeWork if table is stored as RCFile, + * @return MergeWork if table is stored as RCFile or ORCFile, * null otherwise */ - public static MapWork createRCFileMergeTask(FileSinkDesc fsInputDesc, + public static MapWork createMergeTask(FileSinkDesc fsInputDesc, Path finalName, boolean hasDynamicPartitions) throws SemanticException { Path inputDir = fsInputDesc.getFinalDirName(); TableDesc tblDesc = fsInputDesc.getTableInfo(); - if (tblDesc.getInputFileFormatClass().equals(RCFileInputFormat.class)) { + if (tblDesc.getInputFileFormatClass().equals(RCFileInputFormat.class) || + tblDesc.getInputFileFormatClass().equals(OrcInputFormat.class)) { ArrayList inputDirs = new ArrayList(1); ArrayList inputDirstr = new ArrayList(1); if (!hasDynamicPartitions @@ -1497,7 +1509,8 @@ public static MapWork createRCFileMergeTask(FileSinkDesc fsInputDesc, } MergeWork work = new MergeWork(inputDirs, finalName, - hasDynamicPartitions, fsInputDesc.getDynPartCtx()); + hasDynamicPartitions, fsInputDesc.getDynPartCtx(), + tblDesc.getInputFileFormatClass()); LinkedHashMap> pathToAliases = new LinkedHashMap>(); pathToAliases.put(inputDir.toString(), (ArrayList) inputDirstr.clone()); @@ -1515,7 +1528,8 @@ public static MapWork createRCFileMergeTask(FileSinkDesc fsInputDesc, return work; } - throw new SemanticException("createRCFileMergeTask called on non-RCFile table"); + throw new SemanticException("createMergeTask called on a table with file" + + " format other than RCFile or ORCFile"); } /** diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java index 7815ccd..eaf3dc4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.ListBucketingCtx; +import org.apache.hadoop.mapred.InputFormat; @Explain(displayName = "Alter Table Partition Merge Files") public class AlterTablePartMergeFilesDesc { @@ -35,6 +36,7 @@ private List inputDir = new ArrayList(); private Path outputDir = null; + private Class inputFormatClass; public AlterTablePartMergeFilesDesc(String tableName, HashMap partSpec) { @@ -90,4 +92,12 @@ public void setLbCtx(ListBucketingCtx lbCtx) { this.lbCtx = lbCtx; } + public Class getInputFormatClass() { + return inputFormatClass; + } + + public void setInputFormatClass(Class inputFormatClass) { + this.inputFormatClass = inputFormatClass; + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index c0322fb..49f2c3e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -69,6 +69,7 @@ import org.apache.hadoop.hive.ql.index.HiveIndexHandler; import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.io.RCFileInputFormat; +import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager; import org.apache.hadoop.hive.ql.lockmgr.LockException; @@ -1520,11 +1521,13 @@ private void analyzeAlterTablePartMergeFiles(ASTNode tablePartAST, ASTNode ast, tblObj.getSkewedColValueLocationMaps(), tblObj.isStoredAsSubDirectories(), conf); } - // throw a HiveException for non-rcfile. - if (!inputFormatClass.equals(RCFileInputFormat.class)) { + // throw a HiveException for other than rcfile and orcfile. + if (!((inputFormatClass.equals(RCFileInputFormat.class) || + (inputFormatClass.equals(OrcInputFormat.class))))) { throw new SemanticException( - "Only RCFileFormat is supportted right now."); + "Only RCFile and ORCFile Formats are supportted right now."); } + mergeDesc.setInputFormatClass(inputFormatClass); // throw a HiveException if the table/partition is bucketized if (bucketCols != null && bucketCols.size() > 0) { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java index ac34345..deccb16 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java @@ -313,7 +313,8 @@ public void processFileSink(GenTezProcContext context, FileSinkOperator fileSink if (chDir) { // Merge the files in the destination table/partitions by creating Map-only merge job - // If underlying data is RCFile a RCFileBlockMerge task would be created. + // If underlying data is RCFile or OrcFile, RCFileBlockMerge task or + // OrcFileStripeMerge task would be created. LOG.info("using CombineHiveInputformat for the merge job"); GenMapRedUtils.createMRWorkForMergingFiles(fileSink, finalName, context.dependencyTask, context.moveTask, diff --git ql/src/test/queries/clientnegative/orc_merge1.q ql/src/test/queries/clientnegative/orc_merge1.q new file mode 100644 index 0000000..b2d42cd --- /dev/null +++ ql/src/test/queries/clientnegative/orc_merge1.q @@ -0,0 +1,17 @@ +DROP TABLE orcfile_merge; +set hive.exec.dynamic.partition=true; +set hive.exec.dynamic.partition.mode=nonstrict; + +set hive.exec.orc.default.buffer.size=1000; +CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC; + +INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart; + +set hive.exec.orc.default.buffer.size=2000; +INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart; + +-- will fail because of different compression buffer size +ALTER TABLE orcfile_merge PARTITION(ds='2008-04-08',part=11) CONCATENATE; + +DROP TABLE orcfile_merge; diff --git ql/src/test/queries/clientnegative/orc_merge2.q ql/src/test/queries/clientnegative/orc_merge2.q new file mode 100644 index 0000000..2f62ee7 --- /dev/null +++ ql/src/test/queries/clientnegative/orc_merge2.q @@ -0,0 +1,17 @@ +DROP TABLE orcfile_merge; +set hive.exec.dynamic.partition=true; +set hive.exec.dynamic.partition.mode=nonstrict; + +set hive.exec.orc.default.compress=ZLIB; +CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC; + +INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart; + +set hive.exec.orc.default.compress=NONE; +INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart; + +-- will fail because of different compression codec +ALTER TABLE orcfile_merge PARTITION(ds='2008-04-08',part=11) CONCATENATE; + +DROP TABLE orcfile_merge; diff --git ql/src/test/queries/clientnegative/orc_merge3.q ql/src/test/queries/clientnegative/orc_merge3.q new file mode 100644 index 0000000..5158e2e --- /dev/null +++ ql/src/test/queries/clientnegative/orc_merge3.q @@ -0,0 +1,17 @@ +DROP TABLE orcfile_merge; +set hive.exec.dynamic.partition=true; +set hive.exec.dynamic.partition.mode=nonstrict; + +set hive.exec.orc.default.row.index.stride=1000; +CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC; + +INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart; + +set hive.exec.orc.default.row.index.stride=2000; +INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart; + +-- will fail because of different row index stride +ALTER TABLE orcfile_merge PARTITION(ds='2008-04-08',part=11) CONCATENATE; + +DROP TABLE orcfile_merge; diff --git ql/src/test/queries/clientnegative/orc_merge4.q ql/src/test/queries/clientnegative/orc_merge4.q new file mode 100644 index 0000000..ad48572 --- /dev/null +++ ql/src/test/queries/clientnegative/orc_merge4.q @@ -0,0 +1,17 @@ +DROP TABLE orcfile_merge; +set hive.exec.dynamic.partition=true; +set hive.exec.dynamic.partition.mode=nonstrict; + +CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC; + +INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart; + +ALTER TABLE orcfile_merge ADD COLUMNS (newkey int); + +INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT key,value,key,ds,hr FROM srcpart; + +-- will fail because of different column count +ALTER TABLE orcfile_merge PARTITION(ds='2008-04-08',part=11) CONCATENATE; + +DROP TABLE orcfile_merge; diff --git ql/src/test/queries/clientnegative/orc_merge5.q ql/src/test/queries/clientnegative/orc_merge5.q new file mode 100644 index 0000000..e94a8cc --- /dev/null +++ ql/src/test/queries/clientnegative/orc_merge5.q @@ -0,0 +1,17 @@ +DROP TABLE orcfile_merge; +set hive.exec.dynamic.partition=true; +set hive.exec.dynamic.partition.mode=nonstrict; + +set hive.exec.orc.write.format=0.11; +CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC; + +INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart; + +set hive.exec.orc.write.format=0.12; +INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart; + +-- will fail because of different write format +ALTER TABLE orcfile_merge PARTITION(ds='2008-04-08',part=11) CONCATENATE; + +DROP TABLE orcfile_merge; diff --git ql/src/test/queries/clientpositive/alter_merge_2_orc.q ql/src/test/queries/clientpositive/alter_merge_2_orc.q new file mode 100644 index 0000000..580b535 --- /dev/null +++ ql/src/test/queries/clientpositive/alter_merge_2_orc.q @@ -0,0 +1,19 @@ +create table src_orc_merge_test_part(key int, value string) partitioned by (ds string, ts string) stored as orc; + +alter table src_orc_merge_test_part add partition (ds='2012-01-03', ts='2012-01-03+14:46:31'); +desc extended src_orc_merge_test_part partition (ds='2012-01-03', ts='2012-01-03+14:46:31'); + +insert overwrite table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src; +insert into table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src limit 100; +insert into table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src limit 10; + +select count(1) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31'; +select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31'; + +alter table src_orc_merge_test_part partition (ds='2012-01-03', ts='2012-01-03+14:46:31') concatenate; + + +select count(1) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31'; +select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31'; + +drop table src_orc_merge_test_part; diff --git ql/src/test/queries/clientpositive/alter_merge_orc.q ql/src/test/queries/clientpositive/alter_merge_orc.q new file mode 100644 index 0000000..b785766 --- /dev/null +++ ql/src/test/queries/clientpositive/alter_merge_orc.q @@ -0,0 +1,41 @@ +create table src_orc_merge_test(key int, value string) stored as orc; + +insert overwrite table src_orc_merge_test select * from src; +insert into table src_orc_merge_test select * from src limit 100; +insert into table src_orc_merge_test select * from src limit 10; + +show table extended like `src_orc_merge_test`; + +select count(1) from src_orc_merge_test; +select sum(hash(key)), sum(hash(value)) from src_orc_merge_test; + +alter table src_orc_merge_test concatenate; + +show table extended like `src_orc_merge_test`; + +select count(1) from src_orc_merge_test; +select sum(hash(key)), sum(hash(value)) from src_orc_merge_test; + + +create table src_orc_merge_test_part(key int, value string) partitioned by (ds string) stored as orc; + +alter table src_orc_merge_test_part add partition (ds='2011'); + +insert overwrite table src_orc_merge_test_part partition (ds='2011') select * from src; +insert into table src_orc_merge_test_part partition (ds='2011') select * from src limit 100; +insert into table src_orc_merge_test_part partition (ds='2011') select * from src limit 10; + +show table extended like `src_orc_merge_test_part` partition (ds='2011'); + +select count(1) from src_orc_merge_test_part; +select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part; + +alter table src_orc_merge_test_part partition (ds='2011') concatenate; + +show table extended like `src_orc_merge_test_part` partition (ds='2011'); + +select count(1) from src_orc_merge_test_part; +select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part; + +drop table src_orc_merge_test; +drop table src_orc_merge_test_part; diff --git ql/src/test/queries/clientpositive/alter_merge_stats_orc.q ql/src/test/queries/clientpositive/alter_merge_stats_orc.q new file mode 100644 index 0000000..f638d4a --- /dev/null +++ ql/src/test/queries/clientpositive/alter_merge_stats_orc.q @@ -0,0 +1,39 @@ +set hive.exec.submitviachild=true; +set hive.exec.submit.local.task.via.child=true; +create table src_orc_merge_test_stat(key int, value string) stored as orc; + +insert overwrite table src_orc_merge_test_stat select * from src; +insert into table src_orc_merge_test_stat select * from src limit 100; +insert into table src_orc_merge_test_stat select * from src limit 10; + +show table extended like `src_orc_merge_test_stat`; +desc extended src_orc_merge_test_stat; + +analyze table src_orc_merge_test_stat compute statistics noscan; +desc formatted src_orc_merge_test_stat; + +alter table src_orc_merge_test_stat concatenate; +analyze table src_orc_merge_test_stat compute statistics noscan; +desc formatted src_orc_merge_test_stat; + + +create table src_orc_merge_test_part_stat(key int, value string) partitioned by (ds string) stored as orc; + +alter table src_orc_merge_test_part_stat add partition (ds='2011'); + +insert overwrite table src_orc_merge_test_part_stat partition (ds='2011') select * from src; +insert into table src_orc_merge_test_part_stat partition (ds='2011') select * from src limit 100; +insert into table src_orc_merge_test_part_stat partition (ds='2011') select * from src limit 10; + +show table extended like `src_orc_merge_test_part_stat` partition (ds='2011'); +desc formatted src_orc_merge_test_part_stat partition (ds='2011'); + +analyze table src_orc_merge_test_part_stat partition(ds='2011') compute statistics noscan; +desc formatted src_orc_merge_test_part_stat partition (ds='2011'); + +alter table src_orc_merge_test_part_stat partition (ds='2011') concatenate; +analyze table src_orc_merge_test_part_stat partition(ds='2011') compute statistics noscan; +desc formatted src_orc_merge_test_part_stat partition (ds='2011'); + +drop table src_orc_merge_test_stat; +drop table src_orc_merge_test_part_stat; diff --git ql/src/test/queries/clientpositive/orc_merge1.q ql/src/test/queries/clientpositive/orc_merge1.q new file mode 100644 index 0000000..ee65b98 --- /dev/null +++ ql/src/test/queries/clientpositive/orc_merge1.q @@ -0,0 +1,51 @@ +set hive.merge.orcfile.stripe.level=false; +set hive.exec.dynamic.partition=true; +set hive.exec.dynamic.partition.mode=nonstrict; + +DROP TABLE orcfile_merge1; +DROP TABLE orcfile_merge1b; + +CREATE TABLE orcfile_merge1 (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC; +CREATE TABLE orcfile_merge1b (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC; + +-- Use non stipe-level merge +EXPLAIN + INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) + SELECT key, value, PMOD(HASH(key), 100) as part + FROM src; + +INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) + SELECT key, value, PMOD(HASH(key), 100) as part + FROM src; + +DESC FORMATTED orcfile_merge1 partition (ds='1', part='50'); + +set hive.merge.orcfile.stripe.level=true; +EXPLAIN + INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part) + SELECT key, value, PMOD(HASH(key), 100) as part + FROM src; + +INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part) + SELECT key, value, PMOD(HASH(key), 100) as part + FROM src; + +DESC FORMATTED orcfile_merge1 partition (ds='1', part='50'); + +-- Verify +SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) + FROM orcfile_merge1 WHERE ds='1' +) t; + +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; + +SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) + FROM orcfile_merge1b WHERE ds='1' +) t; + +DROP TABLE orcfile_merge1; +DROP TABLE orcfile_merge1b; diff --git ql/src/test/queries/clientpositive/orc_merge2.q ql/src/test/queries/clientpositive/orc_merge2.q new file mode 100644 index 0000000..82115c8 --- /dev/null +++ ql/src/test/queries/clientpositive/orc_merge2.q @@ -0,0 +1,37 @@ +set hive.merge.orcfile.stripe.level=true; +set hive.exec.dynamic.partition=true; +set hive.exec.dynamic.partition.mode=nonstrict; + +DROP TABLE orcfile_merge2a; + +CREATE TABLE orcfile_merge2a (key INT, value STRING) + PARTITIONED BY (one string, two string, three string) + STORED AS ORC; + +EXPLAIN INSERT OVERWRITE TABLE orcfile_merge2a PARTITION (one='1', two, three) + SELECT key, value, PMOD(HASH(key), 10) as two, + PMOD(HASH(value), 10) as three + FROM src; + +INSERT OVERWRITE TABLE orcfile_merge2a PARTITION (one='1', two, three) + SELECT key, value, PMOD(HASH(key), 10) as two, + PMOD(HASH(value), 10) as three + FROM src; + +DESC FORMATTED orcfile_merge2a partition (one='1', two='0', three='2'); + +SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) + FROM orcfile_merge2a +) t; + +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; + +SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value, '1', PMOD(HASH(key), 10), + PMOD(HASH(value), 10)) USING 'tr \t _' AS (c) + FROM src +) t; + +DROP TABLE orcfile_merge2a; + diff --git ql/src/test/queries/clientpositive/orc_merge3.q ql/src/test/queries/clientpositive/orc_merge3.q new file mode 100644 index 0000000..873b3ac --- /dev/null +++ ql/src/test/queries/clientpositive/orc_merge3.q @@ -0,0 +1,37 @@ +set hive.merge.orcfile.stripe.level=true; + +DROP TABLE orcfile_merge3a; +DROP TABLE orcfile_merge3b; + +CREATE TABLE orcfile_merge3a (key int, value string) + PARTITIONED BY (ds string) STORED AS TEXTFILE; +CREATE TABLE orcfile_merge3b (key int, value string) STORED AS ORC; + +INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='1') + SELECT * FROM src; +INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='2') + SELECT * FROM src; + +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +EXPLAIN INSERT OVERWRITE TABLE orcfile_merge3b + SELECT key, value FROM orcfile_merge3a; + +INSERT OVERWRITE TABLE orcfile_merge3b + SELECT key, value FROM orcfile_merge3a; + +DESC FORMATTED orcfile_merge3b; + +SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) + FROM orcfile_merge3a +) t; + +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; + +SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) + FROM orcfile_merge3b +) t; + +DROP TABLE orcfile_merge3a; +DROP TABLE orcfile_merge3b; diff --git ql/src/test/queries/clientpositive/orc_merge4.q ql/src/test/queries/clientpositive/orc_merge4.q new file mode 100644 index 0000000..b5a8e1b --- /dev/null +++ ql/src/test/queries/clientpositive/orc_merge4.q @@ -0,0 +1,47 @@ +set hive.merge.orcfile.stripe.level=true; + +DROP TABLE orcfile_merge3a; +DROP TABLE orcfile_merge3b; + +CREATE TABLE orcfile_merge3a (key int, value string) + PARTITIONED BY (ds string) STORED AS ORC; +CREATE TABLE orcfile_merge3b (key int, value string) STORED AS TEXTFILE; + +set hive.merge.mapfiles=false; +INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='1') + SELECT * FROM src; + +DESC FORMATTED orcfile_merge3a PARTITION (ds='1'); + +set hive.merge.mapfiles=true; +INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='1') + SELECT * FROM src; + +INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='2') + SELECT * FROM src; + +DESC FORMATTED orcfile_merge3a PARTITION (ds='1'); +DESC FORMATTED orcfile_merge3a PARTITION (ds='2'); + +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +EXPLAIN INSERT OVERWRITE TABLE orcfile_merge3b + SELECT key, value FROM orcfile_merge3a; +INSERT OVERWRITE TABLE orcfile_merge3b + SELECT key, value FROM orcfile_merge3a; + +SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) + FROM orcfile_merge3a +) t; + +set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat; +set mapred.max.split.size=100; +set mapref.min.split.size=1; + +SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) + FROM orcfile_merge3b +) t; + +DROP TABLE orcfile_merge3a; +DROP TABLE orcfile_merge3b; diff --git ql/src/test/results/clientnegative/orc_merge1.q.out ql/src/test/results/clientnegative/orc_merge1.q.out new file mode 100644 index 0000000..3ce8403 --- /dev/null +++ ql/src/test/results/clientnegative/orc_merge1.q.out @@ -0,0 +1,74 @@ +PREHOOK: query: DROP TABLE orcfile_merge +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orcfile_merge +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcfile_merge +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@orcfile_merge +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=12 +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@orcfile_merge +POSTHOOK: query: INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=12 +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- will fail because of different compression buffer size +ALTER TABLE orcfile_merge PARTITION(ds='2008-04-08',part=11) CONCATENATE +PREHOOK: type: ALTER_PARTITION_MERGE +PREHOOK: Input: default@orcfile_merge +PREHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +#### A masked pattern was here #### +FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.DDLTask diff --git ql/src/test/results/clientnegative/orc_merge2.q.out ql/src/test/results/clientnegative/orc_merge2.q.out new file mode 100644 index 0000000..3c5a52c --- /dev/null +++ ql/src/test/results/clientnegative/orc_merge2.q.out @@ -0,0 +1,74 @@ +PREHOOK: query: DROP TABLE orcfile_merge +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orcfile_merge +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcfile_merge +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@orcfile_merge +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=12 +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@orcfile_merge +POSTHOOK: query: INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=12 +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- will fail because of different compression codec +ALTER TABLE orcfile_merge PARTITION(ds='2008-04-08',part=11) CONCATENATE +PREHOOK: type: ALTER_PARTITION_MERGE +PREHOOK: Input: default@orcfile_merge +PREHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +#### A masked pattern was here #### +FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.DDLTask diff --git ql/src/test/results/clientnegative/orc_merge3.q.out ql/src/test/results/clientnegative/orc_merge3.q.out new file mode 100644 index 0000000..2895bb0 --- /dev/null +++ ql/src/test/results/clientnegative/orc_merge3.q.out @@ -0,0 +1,74 @@ +PREHOOK: query: DROP TABLE orcfile_merge +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orcfile_merge +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcfile_merge +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@orcfile_merge +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=12 +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@orcfile_merge +POSTHOOK: query: INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=12 +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- will fail because of different row index stride +ALTER TABLE orcfile_merge PARTITION(ds='2008-04-08',part=11) CONCATENATE +PREHOOK: type: ALTER_PARTITION_MERGE +PREHOOK: Input: default@orcfile_merge +PREHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +#### A masked pattern was here #### +FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.DDLTask diff --git ql/src/test/results/clientnegative/orc_merge4.q.out ql/src/test/results/clientnegative/orc_merge4.q.out new file mode 100644 index 0000000..f76e1a3 --- /dev/null +++ ql/src/test/results/clientnegative/orc_merge4.q.out @@ -0,0 +1,86 @@ +PREHOOK: query: DROP TABLE orcfile_merge +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orcfile_merge +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcfile_merge +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@orcfile_merge +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=12 +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE orcfile_merge ADD COLUMNS (newkey int) +PREHOOK: type: ALTERTABLE_ADDCOLS +PREHOOK: Input: default@orcfile_merge +PREHOOK: Output: default@orcfile_merge +POSTHOOK: query: ALTER TABLE orcfile_merge ADD COLUMNS (newkey int) +POSTHOOK: type: ALTERTABLE_ADDCOLS +POSTHOOK: Input: default@orcfile_merge +POSTHOOK: Output: default@orcfile_merge +PREHOOK: query: INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT key,value,key,ds,hr FROM srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@orcfile_merge +POSTHOOK: query: INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT key,value,key,ds,hr FROM srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=12 +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).newkey EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).newkey EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).newkey EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).newkey EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- will fail because of different column count +ALTER TABLE orcfile_merge PARTITION(ds='2008-04-08',part=11) CONCATENATE +PREHOOK: type: ALTER_PARTITION_MERGE +PREHOOK: Input: default@orcfile_merge +PREHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +#### A masked pattern was here #### +FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.DDLTask diff --git ql/src/test/results/clientnegative/orc_merge5.q.out ql/src/test/results/clientnegative/orc_merge5.q.out new file mode 100644 index 0000000..171e46e --- /dev/null +++ ql/src/test/results/clientnegative/orc_merge5.q.out @@ -0,0 +1,74 @@ +PREHOOK: query: DROP TABLE orcfile_merge +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orcfile_merge +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE orcfile_merge (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcfile_merge +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@orcfile_merge +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=12 +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@orcfile_merge +POSTHOOK: query: INSERT INTO TABLE orcfile_merge PARTITION(ds,part) SELECT * FROM srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=12 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=11 +POSTHOOK: Output: default@orcfile_merge@ds=2008-04-09/part=12 +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-08,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge PARTITION(ds=2008-04-09,part=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- will fail because of different write format +ALTER TABLE orcfile_merge PARTITION(ds='2008-04-08',part=11) CONCATENATE +PREHOOK: type: ALTER_PARTITION_MERGE +PREHOOK: Input: default@orcfile_merge +PREHOOK: Output: default@orcfile_merge@ds=2008-04-08/part=11 +#### A masked pattern was here #### +FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.DDLTask diff --git ql/src/test/results/clientpositive/alter_merge_2_orc.q.out ql/src/test/results/clientpositive/alter_merge_2_orc.q.out new file mode 100644 index 0000000..09ede07 --- /dev/null +++ ql/src/test/results/clientpositive/alter_merge_2_orc.q.out @@ -0,0 +1,122 @@ +PREHOOK: query: create table src_orc_merge_test_part(key int, value string) partitioned by (ds string, ts string) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: create table src_orc_merge_test_part(key int, value string) partitioned by (ds string, ts string) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_orc_merge_test_part +PREHOOK: query: alter table src_orc_merge_test_part add partition (ds='2012-01-03', ts='2012-01-03+14:46:31') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@src_orc_merge_test_part +POSTHOOK: query: alter table src_orc_merge_test_part add partition (ds='2012-01-03', ts='2012-01-03+14:46:31') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@src_orc_merge_test_part +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +PREHOOK: query: desc extended src_orc_merge_test_part partition (ds='2012-01-03', ts='2012-01-03+14:46:31') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: query: desc extended src_orc_merge_test_part partition (ds='2012-01-03', ts='2012-01-03+14:46:31') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_orc_merge_test_part +key int +value string +ds string +ts string + +# Partition Information +# col_name data_type comment + +ds string +ts string + +#### A masked pattern was here #### +PREHOOK: query: insert overwrite table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +POSTHOOK: query: insert overwrite table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2012-01-03,ts=2012-01-03+14:46:31).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2012-01-03,ts=2012-01-03+14:46:31).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +POSTHOOK: query: insert into table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2012-01-03,ts=2012-01-03+14:46:31).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2012-01-03,ts=2012-01-03+14:46:31).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +POSTHOOK: query: insert into table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2012-01-03,ts=2012-01-03+14:46:31).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2012-01-03,ts=2012-01-03+14:46:31).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select count(1) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Input: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +#### A masked pattern was here #### +POSTHOOK: query: select count(1) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Input: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +#### A masked pattern was here #### +610 +PREHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Input: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Input: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +#### A masked pattern was here #### +159653 47778930280 +PREHOOK: query: alter table src_orc_merge_test_part partition (ds='2012-01-03', ts='2012-01-03+14:46:31') concatenate +PREHOOK: type: ALTER_PARTITION_MERGE +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +POSTHOOK: query: alter table src_orc_merge_test_part partition (ds='2012-01-03', ts='2012-01-03+14:46:31') concatenate +POSTHOOK: type: ALTER_PARTITION_MERGE +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +PREHOOK: query: select count(1) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Input: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +#### A masked pattern was here #### +POSTHOOK: query: select count(1) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Input: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +#### A masked pattern was here #### +610 +PREHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Input: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Input: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +#### A masked pattern was here #### +159653 47778930280 +PREHOOK: query: drop table src_orc_merge_test_part +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Output: default@src_orc_merge_test_part +POSTHOOK: query: drop table src_orc_merge_test_part +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Output: default@src_orc_merge_test_part diff --git ql/src/test/results/clientpositive/alter_merge_orc.q.out ql/src/test/results/clientpositive/alter_merge_orc.q.out new file mode 100644 index 0000000..270d2bc --- /dev/null +++ ql/src/test/results/clientpositive/alter_merge_orc.q.out @@ -0,0 +1,261 @@ +PREHOOK: query: create table src_orc_merge_test(key int, value string) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: create table src_orc_merge_test(key int, value string) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_orc_merge_test +PREHOOK: query: insert overwrite table src_orc_merge_test select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test +POSTHOOK: query: insert overwrite table src_orc_merge_test select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test +POSTHOOK: Lineage: src_orc_merge_test.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into table src_orc_merge_test select * from src limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test +POSTHOOK: query: insert into table src_orc_merge_test select * from src limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test +POSTHOOK: Lineage: src_orc_merge_test.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into table src_orc_merge_test select * from src limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test +POSTHOOK: query: insert into table src_orc_merge_test select * from src limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test +POSTHOOK: Lineage: src_orc_merge_test.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: show table extended like `src_orc_merge_test` +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like `src_orc_merge_test` +POSTHOOK: type: SHOW_TABLESTATUS +tableName:src_orc_merge_test +#### A masked pattern was here #### +inputformat:org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +outputformat:org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +columns:struct columns { i32 key, string value} +partitioned:false +partitionColumns: +totalNumberFiles:3 +totalFileSize:3570 +maxFileSize:2460 +minFileSize:368 +#### A masked pattern was here #### + +PREHOOK: query: select count(1) from src_orc_merge_test +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test +#### A masked pattern was here #### +POSTHOOK: query: select count(1) from src_orc_merge_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test +#### A masked pattern was here #### +610 +PREHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test +#### A masked pattern was here #### +159653 47778930280 +PREHOOK: query: alter table src_orc_merge_test concatenate +PREHOOK: type: ALTER_TABLE_MERGE +PREHOOK: Input: default@src_orc_merge_test +PREHOOK: Output: default@src_orc_merge_test +POSTHOOK: query: alter table src_orc_merge_test concatenate +POSTHOOK: type: ALTER_TABLE_MERGE +POSTHOOK: Input: default@src_orc_merge_test +POSTHOOK: Output: default@src_orc_merge_test +PREHOOK: query: show table extended like `src_orc_merge_test` +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like `src_orc_merge_test` +POSTHOOK: type: SHOW_TABLESTATUS +tableName:src_orc_merge_test +#### A masked pattern was here #### +inputformat:org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +outputformat:org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +columns:struct columns { i32 key, string value} +partitioned:false +partitionColumns: +totalNumberFiles:1 +totalFileSize:3326 +maxFileSize:3326 +minFileSize:3326 +#### A masked pattern was here #### + +PREHOOK: query: select count(1) from src_orc_merge_test +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test +#### A masked pattern was here #### +POSTHOOK: query: select count(1) from src_orc_merge_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test +#### A masked pattern was here #### +610 +PREHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test +#### A masked pattern was here #### +159653 47778930280 +PREHOOK: query: create table src_orc_merge_test_part(key int, value string) partitioned by (ds string) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: create table src_orc_merge_test_part(key int, value string) partitioned by (ds string) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_orc_merge_test_part +PREHOOK: query: alter table src_orc_merge_test_part add partition (ds='2011') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@src_orc_merge_test_part +POSTHOOK: query: alter table src_orc_merge_test_part add partition (ds='2011') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@src_orc_merge_test_part +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2011 +PREHOOK: query: insert overwrite table src_orc_merge_test_part partition (ds='2011') select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_part@ds=2011 +POSTHOOK: query: insert overwrite table src_orc_merge_test_part partition (ds='2011') select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2011 +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2011).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2011).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into table src_orc_merge_test_part partition (ds='2011') select * from src limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_part@ds=2011 +POSTHOOK: query: insert into table src_orc_merge_test_part partition (ds='2011') select * from src limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2011 +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2011).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2011).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into table src_orc_merge_test_part partition (ds='2011') select * from src limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_part@ds=2011 +POSTHOOK: query: insert into table src_orc_merge_test_part partition (ds='2011') select * from src limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2011 +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2011).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2011).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: show table extended like `src_orc_merge_test_part` partition (ds='2011') +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like `src_orc_merge_test_part` partition (ds='2011') +POSTHOOK: type: SHOW_TABLESTATUS +tableName:src_orc_merge_test_part +#### A masked pattern was here #### +inputformat:org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +outputformat:org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +columns:struct columns { i32 key, string value} +partitioned:true +partitionColumns:struct partition_columns { string ds} +totalNumberFiles:3 +totalFileSize:3570 +maxFileSize:2460 +minFileSize:368 +#### A masked pattern was here #### + +PREHOOK: query: select count(1) from src_orc_merge_test_part +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Input: default@src_orc_merge_test_part@ds=2011 +#### A masked pattern was here #### +POSTHOOK: query: select count(1) from src_orc_merge_test_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Input: default@src_orc_merge_test_part@ds=2011 +#### A masked pattern was here #### +610 +PREHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Input: default@src_orc_merge_test_part@ds=2011 +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Input: default@src_orc_merge_test_part@ds=2011 +#### A masked pattern was here #### +159653 47778930280 +PREHOOK: query: alter table src_orc_merge_test_part partition (ds='2011') concatenate +PREHOOK: type: ALTER_PARTITION_MERGE +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Output: default@src_orc_merge_test_part@ds=2011 +POSTHOOK: query: alter table src_orc_merge_test_part partition (ds='2011') concatenate +POSTHOOK: type: ALTER_PARTITION_MERGE +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2011 +PREHOOK: query: show table extended like `src_orc_merge_test_part` partition (ds='2011') +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like `src_orc_merge_test_part` partition (ds='2011') +POSTHOOK: type: SHOW_TABLESTATUS +tableName:src_orc_merge_test_part +#### A masked pattern was here #### +inputformat:org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +outputformat:org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +columns:struct columns { i32 key, string value} +partitioned:true +partitionColumns:struct partition_columns { string ds} +totalNumberFiles:1 +totalFileSize:3326 +maxFileSize:3326 +minFileSize:3326 +#### A masked pattern was here #### + +PREHOOK: query: select count(1) from src_orc_merge_test_part +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Input: default@src_orc_merge_test_part@ds=2011 +#### A masked pattern was here #### +POSTHOOK: query: select count(1) from src_orc_merge_test_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Input: default@src_orc_merge_test_part@ds=2011 +#### A masked pattern was here #### +610 +PREHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Input: default@src_orc_merge_test_part@ds=2011 +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Input: default@src_orc_merge_test_part@ds=2011 +#### A masked pattern was here #### +159653 47778930280 +PREHOOK: query: drop table src_orc_merge_test +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@src_orc_merge_test +PREHOOK: Output: default@src_orc_merge_test +POSTHOOK: query: drop table src_orc_merge_test +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@src_orc_merge_test +POSTHOOK: Output: default@src_orc_merge_test +PREHOOK: query: drop table src_orc_merge_test_part +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Output: default@src_orc_merge_test_part +POSTHOOK: query: drop table src_orc_merge_test_part +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Output: default@src_orc_merge_test_part diff --git ql/src/test/results/clientpositive/alter_merge_stats_orc.q.out ql/src/test/results/clientpositive/alter_merge_stats_orc.q.out new file mode 100644 index 0000000..bea16df --- /dev/null +++ ql/src/test/results/clientpositive/alter_merge_stats_orc.q.out @@ -0,0 +1,380 @@ +PREHOOK: query: create table src_orc_merge_test_stat(key int, value string) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: create table src_orc_merge_test_stat(key int, value string) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_orc_merge_test_stat +PREHOOK: query: insert overwrite table src_orc_merge_test_stat select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_stat +POSTHOOK: query: insert overwrite table src_orc_merge_test_stat select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_stat +POSTHOOK: Lineage: src_orc_merge_test_stat.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_stat.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into table src_orc_merge_test_stat select * from src limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_stat +POSTHOOK: query: insert into table src_orc_merge_test_stat select * from src limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_stat +POSTHOOK: Lineage: src_orc_merge_test_stat.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_stat.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into table src_orc_merge_test_stat select * from src limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_stat +POSTHOOK: query: insert into table src_orc_merge_test_stat select * from src limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_stat +POSTHOOK: Lineage: src_orc_merge_test_stat.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_stat.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: show table extended like `src_orc_merge_test_stat` +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like `src_orc_merge_test_stat` +POSTHOOK: type: SHOW_TABLESTATUS +tableName:src_orc_merge_test_stat +#### A masked pattern was here #### +inputformat:org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +outputformat:org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +columns:struct columns { i32 key, string value} +partitioned:false +partitionColumns: +totalNumberFiles:3 +totalFileSize:3570 +maxFileSize:2460 +minFileSize:368 +#### A masked pattern was here #### + +PREHOOK: query: desc extended src_orc_merge_test_stat +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_orc_merge_test_stat +POSTHOOK: query: desc extended src_orc_merge_test_stat +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_orc_merge_test_stat +key int +value string + +#### A masked pattern was here #### +PREHOOK: query: analyze table src_orc_merge_test_stat compute statistics noscan +PREHOOK: type: QUERY +PREHOOK: Output: default@src_orc_merge_test_stat +POSTHOOK: query: analyze table src_orc_merge_test_stat compute statistics noscan +POSTHOOK: type: QUERY +POSTHOOK: Output: default@src_orc_merge_test_stat +PREHOOK: query: desc formatted src_orc_merge_test_stat +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_orc_merge_test_stat +POSTHOOK: query: desc formatted src_orc_merge_test_stat +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_orc_merge_test_stat +# col_name data_type comment + +key int +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 3 + numRows 610 + rawDataSize 57340 + totalSize 3570 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: alter table src_orc_merge_test_stat concatenate +PREHOOK: type: ALTER_TABLE_MERGE +PREHOOK: Input: default@src_orc_merge_test_stat +PREHOOK: Output: default@src_orc_merge_test_stat +POSTHOOK: query: alter table src_orc_merge_test_stat concatenate +POSTHOOK: type: ALTER_TABLE_MERGE +POSTHOOK: Input: default@src_orc_merge_test_stat +POSTHOOK: Output: default@src_orc_merge_test_stat +PREHOOK: query: analyze table src_orc_merge_test_stat compute statistics noscan +PREHOOK: type: QUERY +PREHOOK: Output: default@src_orc_merge_test_stat +POSTHOOK: query: analyze table src_orc_merge_test_stat compute statistics noscan +POSTHOOK: type: QUERY +POSTHOOK: Output: default@src_orc_merge_test_stat +PREHOOK: query: desc formatted src_orc_merge_test_stat +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_orc_merge_test_stat +POSTHOOK: query: desc formatted src_orc_merge_test_stat +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_orc_merge_test_stat +# col_name data_type comment + +key int +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 610 + rawDataSize 57340 + totalSize 3326 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: create table src_orc_merge_test_part_stat(key int, value string) partitioned by (ds string) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: create table src_orc_merge_test_part_stat(key int, value string) partitioned by (ds string) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_orc_merge_test_part_stat +PREHOOK: query: alter table src_orc_merge_test_part_stat add partition (ds='2011') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@src_orc_merge_test_part_stat +POSTHOOK: query: alter table src_orc_merge_test_part_stat add partition (ds='2011') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@src_orc_merge_test_part_stat +POSTHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +PREHOOK: query: insert overwrite table src_orc_merge_test_part_stat partition (ds='2011') select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +POSTHOOK: query: insert overwrite table src_orc_merge_test_part_stat partition (ds='2011') select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +POSTHOOK: Lineage: src_orc_merge_test_part_stat PARTITION(ds=2011).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_part_stat PARTITION(ds=2011).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into table src_orc_merge_test_part_stat partition (ds='2011') select * from src limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +POSTHOOK: query: insert into table src_orc_merge_test_part_stat partition (ds='2011') select * from src limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +POSTHOOK: Lineage: src_orc_merge_test_part_stat PARTITION(ds=2011).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_part_stat PARTITION(ds=2011).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into table src_orc_merge_test_part_stat partition (ds='2011') select * from src limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +POSTHOOK: query: insert into table src_orc_merge_test_part_stat partition (ds='2011') select * from src limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +POSTHOOK: Lineage: src_orc_merge_test_part_stat PARTITION(ds=2011).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_part_stat PARTITION(ds=2011).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: show table extended like `src_orc_merge_test_part_stat` partition (ds='2011') +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like `src_orc_merge_test_part_stat` partition (ds='2011') +POSTHOOK: type: SHOW_TABLESTATUS +tableName:src_orc_merge_test_part_stat +#### A masked pattern was here #### +inputformat:org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +outputformat:org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +columns:struct columns { i32 key, string value} +partitioned:true +partitionColumns:struct partition_columns { string ds} +totalNumberFiles:3 +totalFileSize:3570 +maxFileSize:2460 +minFileSize:368 +#### A masked pattern was here #### + +PREHOOK: query: desc formatted src_orc_merge_test_part_stat partition (ds='2011') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_orc_merge_test_part_stat +POSTHOOK: query: desc formatted src_orc_merge_test_part_stat partition (ds='2011') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_orc_merge_test_part_stat +# col_name data_type comment + +key int +value string + +# Partition Information +# col_name data_type comment + +ds string + +# Detailed Partition Information +Partition Value: [2011] +Database: default +Table: src_orc_merge_test_part_stat +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 3 + numRows 10 + rawDataSize 940 + totalSize 3570 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: analyze table src_orc_merge_test_part_stat partition(ds='2011') compute statistics noscan +PREHOOK: type: QUERY +PREHOOK: Output: default@src_orc_merge_test_part_stat +PREHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +POSTHOOK: query: analyze table src_orc_merge_test_part_stat partition(ds='2011') compute statistics noscan +POSTHOOK: type: QUERY +POSTHOOK: Output: default@src_orc_merge_test_part_stat +POSTHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +PREHOOK: query: desc formatted src_orc_merge_test_part_stat partition (ds='2011') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_orc_merge_test_part_stat +POSTHOOK: query: desc formatted src_orc_merge_test_part_stat partition (ds='2011') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_orc_merge_test_part_stat +# col_name data_type comment + +key int +value string + +# Partition Information +# col_name data_type comment + +ds string + +# Detailed Partition Information +Partition Value: [2011] +Database: default +Table: src_orc_merge_test_part_stat +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 3 + numRows 610 + rawDataSize 57340 + totalSize 3570 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: alter table src_orc_merge_test_part_stat partition (ds='2011') concatenate +PREHOOK: type: ALTER_PARTITION_MERGE +PREHOOK: Input: default@src_orc_merge_test_part_stat +PREHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +POSTHOOK: query: alter table src_orc_merge_test_part_stat partition (ds='2011') concatenate +POSTHOOK: type: ALTER_PARTITION_MERGE +POSTHOOK: Input: default@src_orc_merge_test_part_stat +POSTHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +PREHOOK: query: analyze table src_orc_merge_test_part_stat partition(ds='2011') compute statistics noscan +PREHOOK: type: QUERY +PREHOOK: Output: default@src_orc_merge_test_part_stat +PREHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +POSTHOOK: query: analyze table src_orc_merge_test_part_stat partition(ds='2011') compute statistics noscan +POSTHOOK: type: QUERY +POSTHOOK: Output: default@src_orc_merge_test_part_stat +POSTHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +PREHOOK: query: desc formatted src_orc_merge_test_part_stat partition (ds='2011') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_orc_merge_test_part_stat +POSTHOOK: query: desc formatted src_orc_merge_test_part_stat partition (ds='2011') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_orc_merge_test_part_stat +# col_name data_type comment + +key int +value string + +# Partition Information +# col_name data_type comment + +ds string + +# Detailed Partition Information +Partition Value: [2011] +Database: default +Table: src_orc_merge_test_part_stat +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 610 + rawDataSize 57340 + totalSize 3326 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table src_orc_merge_test_stat +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@src_orc_merge_test_stat +PREHOOK: Output: default@src_orc_merge_test_stat +POSTHOOK: query: drop table src_orc_merge_test_stat +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@src_orc_merge_test_stat +POSTHOOK: Output: default@src_orc_merge_test_stat +PREHOOK: query: drop table src_orc_merge_test_part_stat +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@src_orc_merge_test_part_stat +PREHOOK: Output: default@src_orc_merge_test_part_stat +POSTHOOK: query: drop table src_orc_merge_test_part_stat +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@src_orc_merge_test_part_stat +POSTHOOK: Output: default@src_orc_merge_test_part_stat diff --git ql/src/test/results/clientpositive/orc_merge1.q.out ql/src/test/results/clientpositive/orc_merge1.q.out new file mode 100644 index 0000000..e0a1229 --- /dev/null +++ ql/src/test/results/clientpositive/orc_merge1.q.out @@ -0,0 +1,1292 @@ +PREHOOK: query: DROP TABLE orcfile_merge1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orcfile_merge1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE orcfile_merge1b +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orcfile_merge1b +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE orcfile_merge1 (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE orcfile_merge1 (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcfile_merge1 +PREHOOK: query: CREATE TABLE orcfile_merge1b (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE orcfile_merge1b (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcfile_merge1b +PREHOOK: query: -- Use non stipe-level merge +EXPLAIN + INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) + SELECT key, value, PMOD(HASH(key), 100) as part + FROM src +PREHOOK: type: QUERY +POSTHOOK: query: -- Use non stipe-level merge +EXPLAIN + INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) + SELECT key, value, PMOD(HASH(key), 100) as part + FROM src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 100) (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int) + Reduce Operator Tree: + Extract + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1 + + Stage: Stage-0 + Move Operator + tables: + partition: + ds 1 + part + replace: true + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1 + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) + SELECT key, value, PMOD(HASH(key), 100) as part + FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@orcfile_merge1@ds=1 +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) + SELECT key, value, PMOD(HASH(key), 100) as part + FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=0 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=1 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=10 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=11 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=12 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=13 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=14 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=15 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=16 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=17 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=18 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=19 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=2 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=20 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=21 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=22 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=23 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=24 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=25 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=26 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=27 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=28 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=29 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=3 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=30 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=31 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=32 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=33 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=34 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=35 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=36 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=37 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=38 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=39 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=4 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=40 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=41 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=42 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=43 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=44 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=45 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=46 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=47 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=48 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=49 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=5 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=50 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=51 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=52 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=53 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=54 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=55 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=56 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=57 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=58 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=59 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=6 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=60 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=61 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=62 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=63 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=64 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=65 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=66 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=67 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=68 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=69 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=7 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=70 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=71 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=72 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=73 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=74 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=75 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=76 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=77 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=78 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=79 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=8 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=80 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=82 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=83 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=84 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=85 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=86 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=87 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=88 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=89 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=9 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=90 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=91 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=92 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=93 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=94 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=95 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=96 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=97 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=98 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=99 +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=0).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=10).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=10).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=11).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=11).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=12).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=12).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=13).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=13).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=14).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=14).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=15).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=15).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=16).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=16).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=17).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=17).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=18).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=18).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=19).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=19).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=20).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=20).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=21).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=21).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=22).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=22).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=23).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=23).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=24).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=24).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=25).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=25).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=26).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=26).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=27).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=27).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=28).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=28).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=29).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=29).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=30).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=30).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=31).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=31).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=32).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=32).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=33).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=33).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=34).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=34).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=35).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=35).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=36).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=36).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=37).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=37).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=38).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=38).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=39).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=39).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=3).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=3).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=40).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=40).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=41).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=41).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=42).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=42).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=43).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=43).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=44).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=44).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=45).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=45).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=46).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=46).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=47).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=47).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=48).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=48).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=49).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=49).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=4).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=4).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=50).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=50).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=51).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=51).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=52).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=52).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=53).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=53).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=54).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=54).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=55).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=55).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=56).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=56).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=57).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=57).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=58).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=58).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=59).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=59).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=5).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=5).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=60).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=60).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=61).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=61).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=62).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=62).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=63).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=63).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=64).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=64).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=65).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=65).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=66).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=66).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=67).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=67).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=68).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=68).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=69).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=69).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=6).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=6).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=70).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=70).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=71).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=71).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=72).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=72).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=73).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=73).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=74).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=74).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=75).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=75).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=76).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=76).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=77).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=77).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=78).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=78).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=79).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=79).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=7).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=7).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=80).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=80).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=82).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=82).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=83).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=83).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=84).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=84).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=85).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=85).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=86).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=86).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=87).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=87).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=88).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=88).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=89).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=89).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=8).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=8).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=90).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=90).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=91).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=91).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=92).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=92).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=93).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=93).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=94).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=94).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=95).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=95).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=96).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=96).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=97).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=97).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=98).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=98).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=99).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=99).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=9).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=9).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: DESC FORMATTED orcfile_merge1 partition (ds='1', part='50') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@orcfile_merge1 +POSTHOOK: query: DESC FORMATTED orcfile_merge1 partition (ds='1', part='50') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@orcfile_merge1 +# col_name data_type comment + +key int +value string + +# Partition Information +# col_name data_type comment + +ds string +part string + +# Detailed Partition Information +Partition Value: [1, 50] +Database: default +Table: orcfile_merge1 +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 3 + rawDataSize 282 + totalSize 312 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: EXPLAIN + INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part) + SELECT key, value, PMOD(HASH(key), 100) as part + FROM src +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN + INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part) + SELECT key, value, PMOD(HASH(key), 100) as part + FROM src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 100) (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int) + Reduce Operator Tree: + Extract + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1b + + Stage: Stage-0 + Move Operator + tables: + partition: + ds 1 + part + replace: true + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1b + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part) + SELECT key, value, PMOD(HASH(key), 100) as part + FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@orcfile_merge1b@ds=1 +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part) + SELECT key, value, PMOD(HASH(key), 100) as part + FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=0 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=1 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=10 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=11 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=12 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=13 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=14 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=15 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=16 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=17 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=18 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=19 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=2 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=20 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=21 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=22 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=23 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=24 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=25 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=26 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=27 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=28 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=29 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=3 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=30 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=31 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=32 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=33 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=34 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=35 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=36 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=37 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=38 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=39 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=4 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=40 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=41 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=42 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=43 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=44 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=45 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=46 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=47 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=48 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=49 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=5 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=50 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=51 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=52 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=53 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=54 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=55 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=56 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=57 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=58 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=59 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=6 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=60 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=61 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=62 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=63 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=64 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=65 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=66 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=67 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=68 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=69 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=7 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=70 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=71 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=72 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=73 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=74 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=75 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=76 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=77 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=78 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=79 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=8 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=80 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=82 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=83 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=84 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=85 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=86 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=87 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=88 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=89 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=9 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=90 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=91 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=92 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=93 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=94 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=95 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=96 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=97 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=98 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=99 +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=0).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=10).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=10).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=11).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=11).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=12).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=12).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=13).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=13).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=14).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=14).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=15).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=15).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=16).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=16).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=17).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=17).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=18).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=18).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=19).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=19).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=20).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=20).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=21).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=21).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=22).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=22).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=23).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=23).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=24).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=24).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=25).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=25).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=26).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=26).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=27).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=27).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=28).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=28).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=29).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=29).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=30).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=30).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=31).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=31).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=32).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=32).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=33).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=33).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=34).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=34).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=35).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=35).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=36).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=36).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=37).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=37).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=38).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=38).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=39).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=39).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=3).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=3).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=40).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=40).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=41).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=41).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=42).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=42).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=43).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=43).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=44).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=44).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=45).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=45).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=46).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=46).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=47).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=47).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=48).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=48).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=49).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=49).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=4).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=4).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=50).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=50).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=51).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=51).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=52).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=52).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=53).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=53).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=54).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=54).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=55).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=55).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=56).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=56).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=57).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=57).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=58).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=58).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=59).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=59).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=5).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=5).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=60).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=60).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=61).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=61).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=62).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=62).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=63).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=63).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=64).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=64).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=65).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=65).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=66).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=66).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=67).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=67).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=68).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=68).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=69).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=69).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=6).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=6).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=70).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=70).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=71).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=71).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=72).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=72).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=73).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=73).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=74).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=74).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=75).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=75).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=76).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=76).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=77).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=77).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=78).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=78).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=79).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=79).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=7).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=7).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=80).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=80).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=82).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=82).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=83).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=83).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=84).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=84).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=85).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=85).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=86).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=86).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=87).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=87).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=88).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=88).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=89).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=89).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=8).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=8).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=90).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=90).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=91).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=91).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=92).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=92).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=93).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=93).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=94).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=94).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=95).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=95).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=96).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=96).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=97).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=97).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=98).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=98).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=99).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=99).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=9).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=9).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: DESC FORMATTED orcfile_merge1 partition (ds='1', part='50') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@orcfile_merge1 +POSTHOOK: query: DESC FORMATTED orcfile_merge1 partition (ds='1', part='50') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@orcfile_merge1 +# col_name data_type comment + +key int +value string + +# Partition Information +# col_name data_type comment + +ds string +part string + +# Detailed Partition Information +Partition Value: [1, 50] +Database: default +Table: orcfile_merge1 +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 3 + rawDataSize 282 + totalSize 312 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: -- Verify +SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) + FROM orcfile_merge1 WHERE ds='1' +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@orcfile_merge1 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=0 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=1 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=10 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=11 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=12 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=13 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=14 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=15 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=16 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=17 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=18 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=19 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=2 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=20 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=21 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=22 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=23 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=24 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=25 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=26 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=27 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=28 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=29 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=3 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=30 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=31 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=32 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=33 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=34 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=35 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=36 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=37 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=38 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=39 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=4 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=40 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=41 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=42 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=43 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=44 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=45 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=46 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=47 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=48 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=49 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=5 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=50 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=51 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=52 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=53 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=54 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=55 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=56 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=57 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=58 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=59 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=6 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=60 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=61 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=62 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=63 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=64 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=65 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=66 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=67 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=68 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=69 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=7 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=70 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=71 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=72 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=73 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=74 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=75 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=76 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=77 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=78 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=79 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=8 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=80 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=82 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=83 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=84 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=85 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=86 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=87 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=88 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=89 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=9 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=90 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=91 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=92 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=93 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=94 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=95 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=96 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=97 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=98 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=99 +#### A masked pattern was here #### +POSTHOOK: query: -- Verify +SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) + FROM orcfile_merge1 WHERE ds='1' +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orcfile_merge1 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=0 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=1 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=10 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=11 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=12 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=13 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=14 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=15 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=16 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=17 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=18 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=19 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=2 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=20 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=21 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=22 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=23 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=24 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=25 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=26 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=27 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=28 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=29 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=3 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=30 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=31 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=32 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=33 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=34 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=35 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=36 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=37 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=38 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=39 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=4 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=40 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=41 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=42 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=43 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=44 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=45 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=46 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=47 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=48 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=49 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=5 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=50 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=51 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=52 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=53 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=54 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=55 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=56 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=57 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=58 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=59 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=6 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=60 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=61 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=62 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=63 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=64 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=65 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=66 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=67 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=68 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=69 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=7 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=70 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=71 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=72 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=73 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=74 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=75 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=76 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=77 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=78 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=79 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=8 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=80 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=82 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=83 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=84 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=85 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=86 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=87 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=88 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=89 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=9 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=90 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=91 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=92 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=93 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=94 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=95 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=96 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=97 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=98 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=99 +#### A masked pattern was here #### +59521204047 +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) + FROM orcfile_merge1b WHERE ds='1' +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@orcfile_merge1b +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=0 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=1 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=10 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=11 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=12 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=13 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=14 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=15 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=16 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=17 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=18 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=19 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=2 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=20 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=21 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=22 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=23 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=24 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=25 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=26 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=27 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=28 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=29 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=3 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=30 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=31 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=32 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=33 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=34 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=35 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=36 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=37 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=38 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=39 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=4 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=40 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=41 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=42 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=43 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=44 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=45 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=46 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=47 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=48 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=49 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=5 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=50 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=51 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=52 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=53 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=54 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=55 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=56 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=57 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=58 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=59 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=6 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=60 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=61 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=62 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=63 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=64 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=65 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=66 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=67 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=68 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=69 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=7 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=70 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=71 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=72 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=73 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=74 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=75 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=76 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=77 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=78 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=79 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=8 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=80 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=82 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=83 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=84 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=85 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=86 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=87 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=88 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=89 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=9 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=90 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=91 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=92 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=93 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=94 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=95 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=96 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=97 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=98 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=99 +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) + FROM orcfile_merge1b WHERE ds='1' +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orcfile_merge1b +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=0 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=1 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=10 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=11 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=12 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=13 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=14 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=15 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=16 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=17 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=18 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=19 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=2 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=20 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=21 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=22 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=23 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=24 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=25 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=26 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=27 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=28 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=29 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=3 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=30 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=31 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=32 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=33 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=34 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=35 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=36 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=37 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=38 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=39 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=4 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=40 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=41 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=42 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=43 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=44 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=45 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=46 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=47 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=48 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=49 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=5 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=50 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=51 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=52 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=53 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=54 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=55 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=56 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=57 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=58 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=59 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=6 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=60 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=61 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=62 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=63 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=64 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=65 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=66 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=67 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=68 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=69 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=7 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=70 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=71 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=72 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=73 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=74 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=75 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=76 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=77 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=78 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=79 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=8 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=80 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=82 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=83 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=84 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=85 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=86 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=87 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=88 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=89 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=9 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=90 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=91 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=92 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=93 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=94 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=95 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=96 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=97 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=98 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=99 +#### A masked pattern was here #### +59521204047 +PREHOOK: query: DROP TABLE orcfile_merge1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@orcfile_merge1 +PREHOOK: Output: default@orcfile_merge1 +POSTHOOK: query: DROP TABLE orcfile_merge1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@orcfile_merge1 +POSTHOOK: Output: default@orcfile_merge1 +PREHOOK: query: DROP TABLE orcfile_merge1b +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@orcfile_merge1b +PREHOOK: Output: default@orcfile_merge1b +POSTHOOK: query: DROP TABLE orcfile_merge1b +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@orcfile_merge1b +POSTHOOK: Output: default@orcfile_merge1b diff --git ql/src/test/results/clientpositive/orc_merge2.q.out ql/src/test/results/clientpositive/orc_merge2.q.out new file mode 100644 index 0000000..3e4b340 --- /dev/null +++ ql/src/test/results/clientpositive/orc_merge2.q.out @@ -0,0 +1,272 @@ +PREHOOK: query: DROP TABLE orcfile_merge2a +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orcfile_merge2a +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE orcfile_merge2a (key INT, value STRING) + PARTITIONED BY (one string, two string, three string) + STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE orcfile_merge2a (key INT, value STRING) + PARTITIONED BY (one string, two string, three string) + STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcfile_merge2a +PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge2a PARTITION (one='1', two, three) + SELECT key, value, PMOD(HASH(key), 10) as two, + PMOD(HASH(value), 10) as three + FROM src +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge2a PARTITION (one='1', two, three) + SELECT key, value, PMOD(HASH(key), 10) as two, + PMOD(HASH(value), 10) as three + FROM src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 10) (type: int), (hash(value) pmod 10) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int), _col3 (type: int) + sort order: ++ + Map-reduce partition columns: _col2 (type: int), _col3 (type: int) + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int) + Reduce Operator Tree: + Extract + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge2a + + Stage: Stage-0 + Move Operator + tables: + partition: + one 1 + three + two + replace: true + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge2a + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge2a PARTITION (one='1', two, three) + SELECT key, value, PMOD(HASH(key), 10) as two, + PMOD(HASH(value), 10) as three + FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@orcfile_merge2a@one=1 +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge2a PARTITION (one='1', two, three) + SELECT key, value, PMOD(HASH(key), 10) as two, + PMOD(HASH(value), 10) as three + FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=0/three=2 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=0/three=8 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=1/three=3 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=1/three=9 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=2/three=0 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=2/three=4 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=3/three=1 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=3/three=5 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=4/three=2 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=4/three=6 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=5/three=3 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=5/three=7 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=6/three=4 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=6/three=8 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=7/three=5 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=7/three=9 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=8/three=0 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=8/three=6 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=9/three=1 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=9/three=7 +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=0,three=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=0,three=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=0,three=8).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=0,three=8).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=1,three=3).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=1,three=3).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=1,three=9).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=1,three=9).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=2,three=0).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=2,three=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=2,three=4).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=2,three=4).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=3,three=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=3,three=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=3,three=5).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=3,three=5).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=4,three=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=4,three=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=4,three=6).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=4,three=6).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=5,three=3).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=5,three=3).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=5,three=7).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=5,three=7).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=6,three=4).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=6,three=4).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=6,three=8).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=6,three=8).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=7,three=5).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=7,three=5).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=7,three=9).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=7,three=9).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=8,three=0).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=8,three=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=8,three=6).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=8,three=6).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=9,three=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=9,three=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=9,three=7).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=9,three=7).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: DESC FORMATTED orcfile_merge2a partition (one='1', two='0', three='2') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@orcfile_merge2a +POSTHOOK: query: DESC FORMATTED orcfile_merge2a partition (one='1', two='0', three='2') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@orcfile_merge2a +# col_name data_type comment + +key int +value string + +# Partition Information +# col_name data_type comment + +one string +two string +three string + +# Detailed Partition Information +Partition Value: [1, 0, 2] +Database: default +Table: orcfile_merge2a +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 4 + rawDataSize 376 + totalSize 320 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) + FROM orcfile_merge2a +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@orcfile_merge2a +PREHOOK: Input: default@orcfile_merge2a@one=1/two=0/three=2 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=0/three=8 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=1/three=3 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=1/three=9 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=2/three=0 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=2/three=4 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=3/three=1 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=3/three=5 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=4/three=2 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=4/three=6 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=5/three=3 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=5/three=7 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=6/three=4 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=6/three=8 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=7/three=5 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=7/three=9 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=8/three=0 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=8/three=6 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=9/three=1 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=9/three=7 +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) + FROM orcfile_merge2a +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orcfile_merge2a +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=0/three=2 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=0/three=8 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=1/three=3 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=1/three=9 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=2/three=0 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=2/three=4 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=3/three=1 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=3/three=5 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=4/three=2 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=4/three=6 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=5/three=3 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=5/three=7 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=6/three=4 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=6/three=8 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=7/three=5 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=7/three=9 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=8/three=0 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=8/three=6 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=9/three=1 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=9/three=7 +#### A masked pattern was here #### +-4209012844 +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value, '1', PMOD(HASH(key), 10), + PMOD(HASH(value), 10)) USING 'tr \t _' AS (c) + FROM src +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value, '1', PMOD(HASH(key), 10), + PMOD(HASH(value), 10)) USING 'tr \t _' AS (c) + FROM src +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +-4209012844 +PREHOOK: query: DROP TABLE orcfile_merge2a +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@orcfile_merge2a +PREHOOK: Output: default@orcfile_merge2a +POSTHOOK: query: DROP TABLE orcfile_merge2a +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@orcfile_merge2a +POSTHOOK: Output: default@orcfile_merge2a diff --git ql/src/test/results/clientpositive/orc_merge3.q.out ql/src/test/results/clientpositive/orc_merge3.q.out new file mode 100644 index 0000000..93edc38 --- /dev/null +++ ql/src/test/results/clientpositive/orc_merge3.q.out @@ -0,0 +1,224 @@ +PREHOOK: query: DROP TABLE orcfile_merge3a +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orcfile_merge3a +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE orcfile_merge3b +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orcfile_merge3b +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE orcfile_merge3a (key int, value string) + PARTITIONED BY (ds string) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE orcfile_merge3a (key int, value string) + PARTITIONED BY (ds string) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcfile_merge3a +PREHOOK: query: CREATE TABLE orcfile_merge3b (key int, value string) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE orcfile_merge3b (key int, value string) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcfile_merge3b +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='1') + SELECT * FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@orcfile_merge3a@ds=1 +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='1') + SELECT * FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@orcfile_merge3a@ds=1 +POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='2') + SELECT * FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@orcfile_merge3a@ds=2 +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='2') + SELECT * FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@orcfile_merge3a@ds=2 +POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge3b + SELECT key, value FROM orcfile_merge3a +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge3b + SELECT key, value FROM orcfile_merge3a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: orcfile_merge3a + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge3b + + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge3b + + Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 + Merge Work + merge level: stripe + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + + Stage: Stage-5 + Merge Work + merge level: stripe + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3b + SELECT key, value FROM orcfile_merge3a +PREHOOK: type: QUERY +PREHOOK: Input: default@orcfile_merge3a +PREHOOK: Input: default@orcfile_merge3a@ds=1 +PREHOOK: Input: default@orcfile_merge3a@ds=2 +PREHOOK: Output: default@orcfile_merge3b +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3b + SELECT key, value FROM orcfile_merge3a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orcfile_merge3a +POSTHOOK: Input: default@orcfile_merge3a@ds=1 +POSTHOOK: Input: default@orcfile_merge3a@ds=2 +POSTHOOK: Output: default@orcfile_merge3b +POSTHOOK: Lineage: orcfile_merge3b.key SIMPLE [(orcfile_merge3a)orcfile_merge3a.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: orcfile_merge3b.value SIMPLE [(orcfile_merge3a)orcfile_merge3a.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: DESC FORMATTED orcfile_merge3b +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@orcfile_merge3b +POSTHOOK: query: DESC FORMATTED orcfile_merge3b +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@orcfile_merge3b +# col_name data_type comment + +key int +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 1000 + rawDataSize 94000 + totalSize 4762 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) + FROM orcfile_merge3a +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@orcfile_merge3a +PREHOOK: Input: default@orcfile_merge3a@ds=1 +PREHOOK: Input: default@orcfile_merge3a@ds=2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) + FROM orcfile_merge3a +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orcfile_merge3a +POSTHOOK: Input: default@orcfile_merge3a@ds=1 +POSTHOOK: Input: default@orcfile_merge3a@ds=2 +#### A masked pattern was here #### +14412220296 +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) + FROM orcfile_merge3b +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@orcfile_merge3b +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) + FROM orcfile_merge3b +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orcfile_merge3b +#### A masked pattern was here #### +14412220296 +PREHOOK: query: DROP TABLE orcfile_merge3a +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@orcfile_merge3a +PREHOOK: Output: default@orcfile_merge3a +POSTHOOK: query: DROP TABLE orcfile_merge3a +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@orcfile_merge3a +POSTHOOK: Output: default@orcfile_merge3a +PREHOOK: query: DROP TABLE orcfile_merge3b +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@orcfile_merge3b +PREHOOK: Output: default@orcfile_merge3b +POSTHOOK: query: DROP TABLE orcfile_merge3b +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@orcfile_merge3b +POSTHOOK: Output: default@orcfile_merge3b diff --git ql/src/test/results/clientpositive/orc_merge4.q.out ql/src/test/results/clientpositive/orc_merge4.q.out new file mode 100644 index 0000000..423fb3f --- /dev/null +++ ql/src/test/results/clientpositive/orc_merge4.q.out @@ -0,0 +1,337 @@ +PREHOOK: query: DROP TABLE orcfile_merge3a +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orcfile_merge3a +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE orcfile_merge3b +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orcfile_merge3b +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE orcfile_merge3a (key int, value string) + PARTITIONED BY (ds string) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE orcfile_merge3a (key int, value string) + PARTITIONED BY (ds string) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcfile_merge3a +PREHOOK: query: CREATE TABLE orcfile_merge3b (key int, value string) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE orcfile_merge3b (key int, value string) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcfile_merge3b +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='1') + SELECT * FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@orcfile_merge3a@ds=1 +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='1') + SELECT * FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@orcfile_merge3a@ds=1 +POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: DESC FORMATTED orcfile_merge3a PARTITION (ds='1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@orcfile_merge3a +POSTHOOK: query: DESC FORMATTED orcfile_merge3a PARTITION (ds='1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@orcfile_merge3a +# col_name data_type comment + +key int +value string + +# Partition Information +# col_name data_type comment + +ds string + +# Detailed Partition Information +Partition Value: [1] +Database: default +Table: orcfile_merge3a +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 47000 + totalSize 2460 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='1') + SELECT * FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@orcfile_merge3a@ds=1 +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='1') + SELECT * FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@orcfile_merge3a@ds=1 +POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='2') + SELECT * FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@orcfile_merge3a@ds=2 +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='2') + SELECT * FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@orcfile_merge3a@ds=2 +POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: DESC FORMATTED orcfile_merge3a PARTITION (ds='1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@orcfile_merge3a +POSTHOOK: query: DESC FORMATTED orcfile_merge3a PARTITION (ds='1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@orcfile_merge3a +# col_name data_type comment + +key int +value string + +# Partition Information +# col_name data_type comment + +ds string + +# Detailed Partition Information +Partition Value: [1] +Database: default +Table: orcfile_merge3a +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 47000 + totalSize 2460 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: DESC FORMATTED orcfile_merge3a PARTITION (ds='2') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@orcfile_merge3a +POSTHOOK: query: DESC FORMATTED orcfile_merge3a PARTITION (ds='2') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@orcfile_merge3a +# col_name data_type comment + +key int +value string + +# Partition Information +# col_name data_type comment + +ds string + +# Detailed Partition Information +Partition Value: [2] +Database: default +Table: orcfile_merge3a +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 47000 + totalSize 2460 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge3b + SELECT key, value FROM orcfile_merge3a +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge3b + SELECT key, value FROM orcfile_merge3a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: orcfile_merge3a + Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.orcfile_merge3b + + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.orcfile_merge3b + + Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.orcfile_merge3b + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.orcfile_merge3b + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3b + SELECT key, value FROM orcfile_merge3a +PREHOOK: type: QUERY +PREHOOK: Input: default@orcfile_merge3a +PREHOOK: Input: default@orcfile_merge3a@ds=1 +PREHOOK: Input: default@orcfile_merge3a@ds=2 +PREHOOK: Output: default@orcfile_merge3b +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3b + SELECT key, value FROM orcfile_merge3a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orcfile_merge3a +POSTHOOK: Input: default@orcfile_merge3a@ds=1 +POSTHOOK: Input: default@orcfile_merge3a@ds=2 +POSTHOOK: Output: default@orcfile_merge3b +POSTHOOK: Lineage: orcfile_merge3b.key SIMPLE [(orcfile_merge3a)orcfile_merge3a.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: orcfile_merge3b.value SIMPLE [(orcfile_merge3a)orcfile_merge3a.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) + FROM orcfile_merge3a +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@orcfile_merge3a +PREHOOK: Input: default@orcfile_merge3a@ds=1 +PREHOOK: Input: default@orcfile_merge3a@ds=2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) + FROM orcfile_merge3a +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orcfile_merge3a +POSTHOOK: Input: default@orcfile_merge3a@ds=1 +POSTHOOK: Input: default@orcfile_merge3a@ds=2 +#### A masked pattern was here #### +14412220296 +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) + FROM orcfile_merge3b +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@orcfile_merge3b +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) + FROM orcfile_merge3b +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orcfile_merge3b +#### A masked pattern was here #### +14412220296 +PREHOOK: query: DROP TABLE orcfile_merge3a +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@orcfile_merge3a +PREHOOK: Output: default@orcfile_merge3a +POSTHOOK: query: DROP TABLE orcfile_merge3a +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@orcfile_merge3a +POSTHOOK: Output: default@orcfile_merge3a +PREHOOK: query: DROP TABLE orcfile_merge3b +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@orcfile_merge3b +PREHOOK: Output: default@orcfile_merge3b +POSTHOOK: query: DROP TABLE orcfile_merge3b +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@orcfile_merge3b +POSTHOOK: Output: default@orcfile_merge3b diff --git ql/src/test/results/clientpositive/rcfile_merge1.q.out ql/src/test/results/clientpositive/rcfile_merge1.q.out index e49fb51..e8ed908 100644 --- ql/src/test/results/clientpositive/rcfile_merge1.q.out +++ ql/src/test/results/clientpositive/rcfile_merge1.q.out @@ -495,10 +495,14 @@ STAGE PLANS: Stats-Aggr Operator Stage: Stage-3 - Block level merge + Merge Work + merge level: block + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat Stage: Stage-5 - Block level merge + Merge Work + merge level: block + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat Stage: Stage-6 Move Operator diff --git ql/src/test/results/clientpositive/rcfile_merge2.q.out ql/src/test/results/clientpositive/rcfile_merge2.q.out index dfa7eba..7b06efd 100644 --- ql/src/test/results/clientpositive/rcfile_merge2.q.out +++ ql/src/test/results/clientpositive/rcfile_merge2.q.out @@ -80,10 +80,14 @@ STAGE PLANS: Stats-Aggr Operator Stage: Stage-3 - Block level merge + Merge Work + merge level: block + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat Stage: Stage-5 - Block level merge + Merge Work + merge level: block + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat Stage: Stage-6 Move Operator diff --git ql/src/test/results/clientpositive/rcfile_merge3.q.out ql/src/test/results/clientpositive/rcfile_merge3.q.out index a1057da..66e4595 100644 --- ql/src/test/results/clientpositive/rcfile_merge3.q.out +++ ql/src/test/results/clientpositive/rcfile_merge3.q.out @@ -105,10 +105,14 @@ STAGE PLANS: Stats-Aggr Operator Stage: Stage-3 - Block level merge + Merge Work + merge level: block + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat Stage: Stage-5 - Block level merge + Merge Work + merge level: block + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat Stage: Stage-6 Move Operator diff --git ql/src/test/results/clientpositive/tez/alter_merge_2_orc.q.out ql/src/test/results/clientpositive/tez/alter_merge_2_orc.q.out new file mode 100644 index 0000000..09ede07 --- /dev/null +++ ql/src/test/results/clientpositive/tez/alter_merge_2_orc.q.out @@ -0,0 +1,122 @@ +PREHOOK: query: create table src_orc_merge_test_part(key int, value string) partitioned by (ds string, ts string) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: create table src_orc_merge_test_part(key int, value string) partitioned by (ds string, ts string) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_orc_merge_test_part +PREHOOK: query: alter table src_orc_merge_test_part add partition (ds='2012-01-03', ts='2012-01-03+14:46:31') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@src_orc_merge_test_part +POSTHOOK: query: alter table src_orc_merge_test_part add partition (ds='2012-01-03', ts='2012-01-03+14:46:31') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@src_orc_merge_test_part +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +PREHOOK: query: desc extended src_orc_merge_test_part partition (ds='2012-01-03', ts='2012-01-03+14:46:31') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: query: desc extended src_orc_merge_test_part partition (ds='2012-01-03', ts='2012-01-03+14:46:31') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_orc_merge_test_part +key int +value string +ds string +ts string + +# Partition Information +# col_name data_type comment + +ds string +ts string + +#### A masked pattern was here #### +PREHOOK: query: insert overwrite table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +POSTHOOK: query: insert overwrite table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2012-01-03,ts=2012-01-03+14:46:31).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2012-01-03,ts=2012-01-03+14:46:31).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +POSTHOOK: query: insert into table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2012-01-03,ts=2012-01-03+14:46:31).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2012-01-03,ts=2012-01-03+14:46:31).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +POSTHOOK: query: insert into table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2012-01-03,ts=2012-01-03+14:46:31).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2012-01-03,ts=2012-01-03+14:46:31).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select count(1) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Input: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +#### A masked pattern was here #### +POSTHOOK: query: select count(1) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Input: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +#### A masked pattern was here #### +610 +PREHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Input: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Input: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +#### A masked pattern was here #### +159653 47778930280 +PREHOOK: query: alter table src_orc_merge_test_part partition (ds='2012-01-03', ts='2012-01-03+14:46:31') concatenate +PREHOOK: type: ALTER_PARTITION_MERGE +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +POSTHOOK: query: alter table src_orc_merge_test_part partition (ds='2012-01-03', ts='2012-01-03+14:46:31') concatenate +POSTHOOK: type: ALTER_PARTITION_MERGE +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +PREHOOK: query: select count(1) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Input: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +#### A masked pattern was here #### +POSTHOOK: query: select count(1) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Input: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +#### A masked pattern was here #### +610 +PREHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Input: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Input: default@src_orc_merge_test_part@ds=2012-01-03/ts=2012-01-03+14%3A46%3A31 +#### A masked pattern was here #### +159653 47778930280 +PREHOOK: query: drop table src_orc_merge_test_part +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Output: default@src_orc_merge_test_part +POSTHOOK: query: drop table src_orc_merge_test_part +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Output: default@src_orc_merge_test_part diff --git ql/src/test/results/clientpositive/tez/alter_merge_orc.q.out ql/src/test/results/clientpositive/tez/alter_merge_orc.q.out new file mode 100644 index 0000000..270d2bc --- /dev/null +++ ql/src/test/results/clientpositive/tez/alter_merge_orc.q.out @@ -0,0 +1,261 @@ +PREHOOK: query: create table src_orc_merge_test(key int, value string) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: create table src_orc_merge_test(key int, value string) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_orc_merge_test +PREHOOK: query: insert overwrite table src_orc_merge_test select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test +POSTHOOK: query: insert overwrite table src_orc_merge_test select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test +POSTHOOK: Lineage: src_orc_merge_test.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into table src_orc_merge_test select * from src limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test +POSTHOOK: query: insert into table src_orc_merge_test select * from src limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test +POSTHOOK: Lineage: src_orc_merge_test.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into table src_orc_merge_test select * from src limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test +POSTHOOK: query: insert into table src_orc_merge_test select * from src limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test +POSTHOOK: Lineage: src_orc_merge_test.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: show table extended like `src_orc_merge_test` +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like `src_orc_merge_test` +POSTHOOK: type: SHOW_TABLESTATUS +tableName:src_orc_merge_test +#### A masked pattern was here #### +inputformat:org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +outputformat:org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +columns:struct columns { i32 key, string value} +partitioned:false +partitionColumns: +totalNumberFiles:3 +totalFileSize:3570 +maxFileSize:2460 +minFileSize:368 +#### A masked pattern was here #### + +PREHOOK: query: select count(1) from src_orc_merge_test +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test +#### A masked pattern was here #### +POSTHOOK: query: select count(1) from src_orc_merge_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test +#### A masked pattern was here #### +610 +PREHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test +#### A masked pattern was here #### +159653 47778930280 +PREHOOK: query: alter table src_orc_merge_test concatenate +PREHOOK: type: ALTER_TABLE_MERGE +PREHOOK: Input: default@src_orc_merge_test +PREHOOK: Output: default@src_orc_merge_test +POSTHOOK: query: alter table src_orc_merge_test concatenate +POSTHOOK: type: ALTER_TABLE_MERGE +POSTHOOK: Input: default@src_orc_merge_test +POSTHOOK: Output: default@src_orc_merge_test +PREHOOK: query: show table extended like `src_orc_merge_test` +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like `src_orc_merge_test` +POSTHOOK: type: SHOW_TABLESTATUS +tableName:src_orc_merge_test +#### A masked pattern was here #### +inputformat:org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +outputformat:org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +columns:struct columns { i32 key, string value} +partitioned:false +partitionColumns: +totalNumberFiles:1 +totalFileSize:3326 +maxFileSize:3326 +minFileSize:3326 +#### A masked pattern was here #### + +PREHOOK: query: select count(1) from src_orc_merge_test +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test +#### A masked pattern was here #### +POSTHOOK: query: select count(1) from src_orc_merge_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test +#### A masked pattern was here #### +610 +PREHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test +#### A masked pattern was here #### +159653 47778930280 +PREHOOK: query: create table src_orc_merge_test_part(key int, value string) partitioned by (ds string) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: create table src_orc_merge_test_part(key int, value string) partitioned by (ds string) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_orc_merge_test_part +PREHOOK: query: alter table src_orc_merge_test_part add partition (ds='2011') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@src_orc_merge_test_part +POSTHOOK: query: alter table src_orc_merge_test_part add partition (ds='2011') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@src_orc_merge_test_part +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2011 +PREHOOK: query: insert overwrite table src_orc_merge_test_part partition (ds='2011') select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_part@ds=2011 +POSTHOOK: query: insert overwrite table src_orc_merge_test_part partition (ds='2011') select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2011 +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2011).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2011).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into table src_orc_merge_test_part partition (ds='2011') select * from src limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_part@ds=2011 +POSTHOOK: query: insert into table src_orc_merge_test_part partition (ds='2011') select * from src limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2011 +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2011).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2011).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into table src_orc_merge_test_part partition (ds='2011') select * from src limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_part@ds=2011 +POSTHOOK: query: insert into table src_orc_merge_test_part partition (ds='2011') select * from src limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2011 +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2011).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_part PARTITION(ds=2011).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: show table extended like `src_orc_merge_test_part` partition (ds='2011') +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like `src_orc_merge_test_part` partition (ds='2011') +POSTHOOK: type: SHOW_TABLESTATUS +tableName:src_orc_merge_test_part +#### A masked pattern was here #### +inputformat:org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +outputformat:org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +columns:struct columns { i32 key, string value} +partitioned:true +partitionColumns:struct partition_columns { string ds} +totalNumberFiles:3 +totalFileSize:3570 +maxFileSize:2460 +minFileSize:368 +#### A masked pattern was here #### + +PREHOOK: query: select count(1) from src_orc_merge_test_part +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Input: default@src_orc_merge_test_part@ds=2011 +#### A masked pattern was here #### +POSTHOOK: query: select count(1) from src_orc_merge_test_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Input: default@src_orc_merge_test_part@ds=2011 +#### A masked pattern was here #### +610 +PREHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Input: default@src_orc_merge_test_part@ds=2011 +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Input: default@src_orc_merge_test_part@ds=2011 +#### A masked pattern was here #### +159653 47778930280 +PREHOOK: query: alter table src_orc_merge_test_part partition (ds='2011') concatenate +PREHOOK: type: ALTER_PARTITION_MERGE +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Output: default@src_orc_merge_test_part@ds=2011 +POSTHOOK: query: alter table src_orc_merge_test_part partition (ds='2011') concatenate +POSTHOOK: type: ALTER_PARTITION_MERGE +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Output: default@src_orc_merge_test_part@ds=2011 +PREHOOK: query: show table extended like `src_orc_merge_test_part` partition (ds='2011') +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like `src_orc_merge_test_part` partition (ds='2011') +POSTHOOK: type: SHOW_TABLESTATUS +tableName:src_orc_merge_test_part +#### A masked pattern was here #### +inputformat:org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +outputformat:org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +columns:struct columns { i32 key, string value} +partitioned:true +partitionColumns:struct partition_columns { string ds} +totalNumberFiles:1 +totalFileSize:3326 +maxFileSize:3326 +minFileSize:3326 +#### A masked pattern was here #### + +PREHOOK: query: select count(1) from src_orc_merge_test_part +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Input: default@src_orc_merge_test_part@ds=2011 +#### A masked pattern was here #### +POSTHOOK: query: select count(1) from src_orc_merge_test_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Input: default@src_orc_merge_test_part@ds=2011 +#### A masked pattern was here #### +610 +PREHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part +PREHOOK: type: QUERY +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Input: default@src_orc_merge_test_part@ds=2011 +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Input: default@src_orc_merge_test_part@ds=2011 +#### A masked pattern was here #### +159653 47778930280 +PREHOOK: query: drop table src_orc_merge_test +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@src_orc_merge_test +PREHOOK: Output: default@src_orc_merge_test +POSTHOOK: query: drop table src_orc_merge_test +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@src_orc_merge_test +POSTHOOK: Output: default@src_orc_merge_test +PREHOOK: query: drop table src_orc_merge_test_part +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@src_orc_merge_test_part +PREHOOK: Output: default@src_orc_merge_test_part +POSTHOOK: query: drop table src_orc_merge_test_part +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@src_orc_merge_test_part +POSTHOOK: Output: default@src_orc_merge_test_part diff --git ql/src/test/results/clientpositive/tez/alter_merge_stats_orc.q.out ql/src/test/results/clientpositive/tez/alter_merge_stats_orc.q.out new file mode 100644 index 0000000..bea16df --- /dev/null +++ ql/src/test/results/clientpositive/tez/alter_merge_stats_orc.q.out @@ -0,0 +1,380 @@ +PREHOOK: query: create table src_orc_merge_test_stat(key int, value string) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: create table src_orc_merge_test_stat(key int, value string) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_orc_merge_test_stat +PREHOOK: query: insert overwrite table src_orc_merge_test_stat select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_stat +POSTHOOK: query: insert overwrite table src_orc_merge_test_stat select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_stat +POSTHOOK: Lineage: src_orc_merge_test_stat.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_stat.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into table src_orc_merge_test_stat select * from src limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_stat +POSTHOOK: query: insert into table src_orc_merge_test_stat select * from src limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_stat +POSTHOOK: Lineage: src_orc_merge_test_stat.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_stat.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into table src_orc_merge_test_stat select * from src limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_stat +POSTHOOK: query: insert into table src_orc_merge_test_stat select * from src limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_stat +POSTHOOK: Lineage: src_orc_merge_test_stat.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_stat.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: show table extended like `src_orc_merge_test_stat` +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like `src_orc_merge_test_stat` +POSTHOOK: type: SHOW_TABLESTATUS +tableName:src_orc_merge_test_stat +#### A masked pattern was here #### +inputformat:org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +outputformat:org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +columns:struct columns { i32 key, string value} +partitioned:false +partitionColumns: +totalNumberFiles:3 +totalFileSize:3570 +maxFileSize:2460 +minFileSize:368 +#### A masked pattern was here #### + +PREHOOK: query: desc extended src_orc_merge_test_stat +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_orc_merge_test_stat +POSTHOOK: query: desc extended src_orc_merge_test_stat +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_orc_merge_test_stat +key int +value string + +#### A masked pattern was here #### +PREHOOK: query: analyze table src_orc_merge_test_stat compute statistics noscan +PREHOOK: type: QUERY +PREHOOK: Output: default@src_orc_merge_test_stat +POSTHOOK: query: analyze table src_orc_merge_test_stat compute statistics noscan +POSTHOOK: type: QUERY +POSTHOOK: Output: default@src_orc_merge_test_stat +PREHOOK: query: desc formatted src_orc_merge_test_stat +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_orc_merge_test_stat +POSTHOOK: query: desc formatted src_orc_merge_test_stat +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_orc_merge_test_stat +# col_name data_type comment + +key int +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 3 + numRows 610 + rawDataSize 57340 + totalSize 3570 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: alter table src_orc_merge_test_stat concatenate +PREHOOK: type: ALTER_TABLE_MERGE +PREHOOK: Input: default@src_orc_merge_test_stat +PREHOOK: Output: default@src_orc_merge_test_stat +POSTHOOK: query: alter table src_orc_merge_test_stat concatenate +POSTHOOK: type: ALTER_TABLE_MERGE +POSTHOOK: Input: default@src_orc_merge_test_stat +POSTHOOK: Output: default@src_orc_merge_test_stat +PREHOOK: query: analyze table src_orc_merge_test_stat compute statistics noscan +PREHOOK: type: QUERY +PREHOOK: Output: default@src_orc_merge_test_stat +POSTHOOK: query: analyze table src_orc_merge_test_stat compute statistics noscan +POSTHOOK: type: QUERY +POSTHOOK: Output: default@src_orc_merge_test_stat +PREHOOK: query: desc formatted src_orc_merge_test_stat +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_orc_merge_test_stat +POSTHOOK: query: desc formatted src_orc_merge_test_stat +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_orc_merge_test_stat +# col_name data_type comment + +key int +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 610 + rawDataSize 57340 + totalSize 3326 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: create table src_orc_merge_test_part_stat(key int, value string) partitioned by (ds string) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: create table src_orc_merge_test_part_stat(key int, value string) partitioned by (ds string) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_orc_merge_test_part_stat +PREHOOK: query: alter table src_orc_merge_test_part_stat add partition (ds='2011') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@src_orc_merge_test_part_stat +POSTHOOK: query: alter table src_orc_merge_test_part_stat add partition (ds='2011') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@src_orc_merge_test_part_stat +POSTHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +PREHOOK: query: insert overwrite table src_orc_merge_test_part_stat partition (ds='2011') select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +POSTHOOK: query: insert overwrite table src_orc_merge_test_part_stat partition (ds='2011') select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +POSTHOOK: Lineage: src_orc_merge_test_part_stat PARTITION(ds=2011).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_part_stat PARTITION(ds=2011).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into table src_orc_merge_test_part_stat partition (ds='2011') select * from src limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +POSTHOOK: query: insert into table src_orc_merge_test_part_stat partition (ds='2011') select * from src limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +POSTHOOK: Lineage: src_orc_merge_test_part_stat PARTITION(ds=2011).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_part_stat PARTITION(ds=2011).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into table src_orc_merge_test_part_stat partition (ds='2011') select * from src limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +POSTHOOK: query: insert into table src_orc_merge_test_part_stat partition (ds='2011') select * from src limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +POSTHOOK: Lineage: src_orc_merge_test_part_stat PARTITION(ds=2011).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_orc_merge_test_part_stat PARTITION(ds=2011).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: show table extended like `src_orc_merge_test_part_stat` partition (ds='2011') +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like `src_orc_merge_test_part_stat` partition (ds='2011') +POSTHOOK: type: SHOW_TABLESTATUS +tableName:src_orc_merge_test_part_stat +#### A masked pattern was here #### +inputformat:org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +outputformat:org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +columns:struct columns { i32 key, string value} +partitioned:true +partitionColumns:struct partition_columns { string ds} +totalNumberFiles:3 +totalFileSize:3570 +maxFileSize:2460 +minFileSize:368 +#### A masked pattern was here #### + +PREHOOK: query: desc formatted src_orc_merge_test_part_stat partition (ds='2011') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_orc_merge_test_part_stat +POSTHOOK: query: desc formatted src_orc_merge_test_part_stat partition (ds='2011') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_orc_merge_test_part_stat +# col_name data_type comment + +key int +value string + +# Partition Information +# col_name data_type comment + +ds string + +# Detailed Partition Information +Partition Value: [2011] +Database: default +Table: src_orc_merge_test_part_stat +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 3 + numRows 10 + rawDataSize 940 + totalSize 3570 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: analyze table src_orc_merge_test_part_stat partition(ds='2011') compute statistics noscan +PREHOOK: type: QUERY +PREHOOK: Output: default@src_orc_merge_test_part_stat +PREHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +POSTHOOK: query: analyze table src_orc_merge_test_part_stat partition(ds='2011') compute statistics noscan +POSTHOOK: type: QUERY +POSTHOOK: Output: default@src_orc_merge_test_part_stat +POSTHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +PREHOOK: query: desc formatted src_orc_merge_test_part_stat partition (ds='2011') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_orc_merge_test_part_stat +POSTHOOK: query: desc formatted src_orc_merge_test_part_stat partition (ds='2011') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_orc_merge_test_part_stat +# col_name data_type comment + +key int +value string + +# Partition Information +# col_name data_type comment + +ds string + +# Detailed Partition Information +Partition Value: [2011] +Database: default +Table: src_orc_merge_test_part_stat +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 3 + numRows 610 + rawDataSize 57340 + totalSize 3570 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: alter table src_orc_merge_test_part_stat partition (ds='2011') concatenate +PREHOOK: type: ALTER_PARTITION_MERGE +PREHOOK: Input: default@src_orc_merge_test_part_stat +PREHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +POSTHOOK: query: alter table src_orc_merge_test_part_stat partition (ds='2011') concatenate +POSTHOOK: type: ALTER_PARTITION_MERGE +POSTHOOK: Input: default@src_orc_merge_test_part_stat +POSTHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +PREHOOK: query: analyze table src_orc_merge_test_part_stat partition(ds='2011') compute statistics noscan +PREHOOK: type: QUERY +PREHOOK: Output: default@src_orc_merge_test_part_stat +PREHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +POSTHOOK: query: analyze table src_orc_merge_test_part_stat partition(ds='2011') compute statistics noscan +POSTHOOK: type: QUERY +POSTHOOK: Output: default@src_orc_merge_test_part_stat +POSTHOOK: Output: default@src_orc_merge_test_part_stat@ds=2011 +PREHOOK: query: desc formatted src_orc_merge_test_part_stat partition (ds='2011') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_orc_merge_test_part_stat +POSTHOOK: query: desc formatted src_orc_merge_test_part_stat partition (ds='2011') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_orc_merge_test_part_stat +# col_name data_type comment + +key int +value string + +# Partition Information +# col_name data_type comment + +ds string + +# Detailed Partition Information +Partition Value: [2011] +Database: default +Table: src_orc_merge_test_part_stat +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 610 + rawDataSize 57340 + totalSize 3326 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table src_orc_merge_test_stat +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@src_orc_merge_test_stat +PREHOOK: Output: default@src_orc_merge_test_stat +POSTHOOK: query: drop table src_orc_merge_test_stat +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@src_orc_merge_test_stat +POSTHOOK: Output: default@src_orc_merge_test_stat +PREHOOK: query: drop table src_orc_merge_test_part_stat +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@src_orc_merge_test_part_stat +PREHOOK: Output: default@src_orc_merge_test_part_stat +POSTHOOK: query: drop table src_orc_merge_test_part_stat +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@src_orc_merge_test_part_stat +POSTHOOK: Output: default@src_orc_merge_test_part_stat diff --git ql/src/test/results/clientpositive/tez/orc_merge1.q.out ql/src/test/results/clientpositive/tez/orc_merge1.q.out new file mode 100644 index 0000000..62c316f --- /dev/null +++ ql/src/test/results/clientpositive/tez/orc_merge1.q.out @@ -0,0 +1,1312 @@ +PREHOOK: query: DROP TABLE orcfile_merge1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orcfile_merge1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE orcfile_merge1b +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orcfile_merge1b +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE orcfile_merge1 (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE orcfile_merge1 (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcfile_merge1 +PREHOOK: query: CREATE TABLE orcfile_merge1b (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE orcfile_merge1b (key INT, value STRING) + PARTITIONED BY (ds STRING, part STRING) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcfile_merge1b +PREHOOK: query: -- Use non stipe-level merge +EXPLAIN + INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) + SELECT key, value, PMOD(HASH(key), 100) as part + FROM src +PREHOOK: type: QUERY +POSTHOOK: query: -- Use non stipe-level merge +EXPLAIN + INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) + SELECT key, value, PMOD(HASH(key), 100) as part + FROM src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 100) (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int) + Reducer 2 + Reduce Operator Tree: + Extract + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds 1 + part + replace: true + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1 + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) + SELECT key, value, PMOD(HASH(key), 100) as part + FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@orcfile_merge1@ds=1 +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) + SELECT key, value, PMOD(HASH(key), 100) as part + FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=0 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=1 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=10 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=11 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=12 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=13 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=14 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=15 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=16 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=17 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=18 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=19 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=2 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=20 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=21 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=22 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=23 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=24 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=25 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=26 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=27 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=28 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=29 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=3 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=30 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=31 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=32 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=33 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=34 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=35 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=36 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=37 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=38 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=39 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=4 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=40 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=41 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=42 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=43 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=44 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=45 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=46 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=47 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=48 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=49 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=5 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=50 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=51 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=52 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=53 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=54 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=55 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=56 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=57 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=58 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=59 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=6 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=60 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=61 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=62 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=63 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=64 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=65 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=66 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=67 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=68 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=69 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=7 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=70 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=71 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=72 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=73 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=74 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=75 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=76 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=77 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=78 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=79 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=8 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=80 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=82 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=83 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=84 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=85 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=86 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=87 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=88 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=89 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=9 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=90 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=91 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=92 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=93 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=94 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=95 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=96 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=97 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=98 +POSTHOOK: Output: default@orcfile_merge1@ds=1/part=99 +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=0).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=10).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=10).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=11).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=11).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=12).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=12).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=13).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=13).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=14).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=14).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=15).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=15).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=16).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=16).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=17).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=17).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=18).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=18).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=19).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=19).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=20).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=20).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=21).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=21).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=22).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=22).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=23).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=23).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=24).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=24).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=25).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=25).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=26).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=26).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=27).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=27).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=28).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=28).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=29).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=29).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=30).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=30).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=31).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=31).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=32).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=32).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=33).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=33).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=34).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=34).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=35).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=35).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=36).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=36).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=37).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=37).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=38).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=38).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=39).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=39).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=3).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=3).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=40).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=40).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=41).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=41).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=42).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=42).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=43).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=43).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=44).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=44).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=45).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=45).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=46).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=46).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=47).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=47).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=48).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=48).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=49).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=49).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=4).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=4).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=50).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=50).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=51).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=51).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=52).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=52).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=53).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=53).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=54).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=54).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=55).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=55).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=56).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=56).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=57).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=57).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=58).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=58).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=59).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=59).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=5).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=5).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=60).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=60).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=61).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=61).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=62).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=62).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=63).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=63).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=64).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=64).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=65).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=65).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=66).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=66).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=67).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=67).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=68).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=68).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=69).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=69).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=6).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=6).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=70).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=70).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=71).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=71).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=72).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=72).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=73).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=73).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=74).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=74).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=75).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=75).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=76).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=76).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=77).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=77).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=78).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=78).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=79).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=79).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=7).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=7).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=80).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=80).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=82).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=82).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=83).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=83).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=84).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=84).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=85).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=85).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=86).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=86).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=87).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=87).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=88).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=88).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=89).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=89).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=8).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=8).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=90).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=90).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=91).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=91).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=92).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=92).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=93).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=93).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=94).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=94).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=95).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=95).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=96).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=96).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=97).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=97).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=98).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=98).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=99).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=99).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=9).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=9).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: DESC FORMATTED orcfile_merge1 partition (ds='1', part='50') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@orcfile_merge1 +POSTHOOK: query: DESC FORMATTED orcfile_merge1 partition (ds='1', part='50') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@orcfile_merge1 +# col_name data_type comment + +key int +value string + +# Partition Information +# col_name data_type comment + +ds string +part string + +# Detailed Partition Information +Partition Value: [1, 50] +Database: default +Table: orcfile_merge1 +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 3 + rawDataSize 282 + totalSize 312 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: EXPLAIN + INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part) + SELECT key, value, PMOD(HASH(key), 100) as part + FROM src +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN + INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part) + SELECT key, value, PMOD(HASH(key), 100) as part + FROM src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 100) (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int) + Reducer 2 + Reduce Operator Tree: + Extract + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1b + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds 1 + part + replace: true + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1b + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part) + SELECT key, value, PMOD(HASH(key), 100) as part + FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@orcfile_merge1b@ds=1 +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part) + SELECT key, value, PMOD(HASH(key), 100) as part + FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=0 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=1 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=10 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=11 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=12 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=13 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=14 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=15 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=16 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=17 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=18 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=19 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=2 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=20 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=21 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=22 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=23 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=24 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=25 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=26 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=27 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=28 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=29 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=3 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=30 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=31 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=32 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=33 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=34 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=35 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=36 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=37 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=38 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=39 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=4 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=40 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=41 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=42 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=43 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=44 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=45 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=46 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=47 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=48 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=49 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=5 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=50 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=51 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=52 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=53 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=54 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=55 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=56 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=57 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=58 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=59 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=6 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=60 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=61 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=62 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=63 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=64 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=65 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=66 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=67 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=68 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=69 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=7 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=70 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=71 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=72 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=73 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=74 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=75 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=76 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=77 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=78 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=79 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=8 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=80 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=82 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=83 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=84 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=85 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=86 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=87 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=88 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=89 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=9 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=90 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=91 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=92 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=93 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=94 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=95 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=96 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=97 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=98 +POSTHOOK: Output: default@orcfile_merge1b@ds=1/part=99 +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=0).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=10).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=10).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=11).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=11).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=12).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=12).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=13).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=13).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=14).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=14).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=15).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=15).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=16).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=16).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=17).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=17).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=18).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=18).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=19).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=19).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=20).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=20).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=21).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=21).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=22).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=22).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=23).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=23).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=24).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=24).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=25).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=25).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=26).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=26).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=27).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=27).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=28).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=28).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=29).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=29).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=30).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=30).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=31).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=31).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=32).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=32).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=33).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=33).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=34).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=34).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=35).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=35).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=36).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=36).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=37).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=37).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=38).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=38).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=39).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=39).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=3).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=3).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=40).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=40).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=41).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=41).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=42).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=42).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=43).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=43).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=44).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=44).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=45).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=45).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=46).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=46).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=47).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=47).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=48).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=48).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=49).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=49).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=4).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=4).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=50).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=50).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=51).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=51).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=52).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=52).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=53).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=53).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=54).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=54).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=55).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=55).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=56).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=56).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=57).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=57).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=58).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=58).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=59).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=59).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=5).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=5).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=60).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=60).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=61).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=61).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=62).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=62).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=63).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=63).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=64).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=64).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=65).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=65).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=66).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=66).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=67).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=67).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=68).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=68).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=69).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=69).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=6).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=6).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=70).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=70).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=71).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=71).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=72).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=72).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=73).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=73).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=74).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=74).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=75).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=75).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=76).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=76).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=77).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=77).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=78).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=78).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=79).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=79).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=7).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=7).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=80).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=80).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=82).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=82).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=83).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=83).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=84).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=84).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=85).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=85).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=86).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=86).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=87).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=87).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=88).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=88).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=89).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=89).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=8).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=8).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=90).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=90).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=91).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=91).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=92).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=92).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=93).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=93).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=94).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=94).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=95).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=95).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=96).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=96).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=97).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=97).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=98).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=98).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=99).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=99).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=9).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=9).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: DESC FORMATTED orcfile_merge1 partition (ds='1', part='50') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@orcfile_merge1 +POSTHOOK: query: DESC FORMATTED orcfile_merge1 partition (ds='1', part='50') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@orcfile_merge1 +# col_name data_type comment + +key int +value string + +# Partition Information +# col_name data_type comment + +ds string +part string + +# Detailed Partition Information +Partition Value: [1, 50] +Database: default +Table: orcfile_merge1 +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 3 + rawDataSize 282 + totalSize 312 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: -- Verify +SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) + FROM orcfile_merge1 WHERE ds='1' +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@orcfile_merge1 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=0 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=1 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=10 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=11 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=12 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=13 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=14 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=15 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=16 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=17 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=18 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=19 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=2 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=20 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=21 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=22 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=23 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=24 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=25 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=26 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=27 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=28 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=29 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=3 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=30 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=31 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=32 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=33 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=34 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=35 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=36 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=37 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=38 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=39 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=4 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=40 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=41 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=42 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=43 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=44 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=45 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=46 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=47 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=48 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=49 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=5 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=50 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=51 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=52 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=53 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=54 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=55 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=56 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=57 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=58 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=59 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=6 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=60 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=61 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=62 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=63 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=64 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=65 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=66 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=67 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=68 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=69 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=7 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=70 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=71 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=72 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=73 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=74 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=75 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=76 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=77 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=78 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=79 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=8 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=80 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=82 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=83 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=84 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=85 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=86 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=87 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=88 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=89 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=9 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=90 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=91 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=92 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=93 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=94 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=95 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=96 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=97 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=98 +PREHOOK: Input: default@orcfile_merge1@ds=1/part=99 +#### A masked pattern was here #### +POSTHOOK: query: -- Verify +SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) + FROM orcfile_merge1 WHERE ds='1' +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orcfile_merge1 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=0 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=1 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=10 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=11 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=12 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=13 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=14 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=15 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=16 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=17 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=18 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=19 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=2 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=20 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=21 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=22 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=23 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=24 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=25 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=26 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=27 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=28 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=29 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=3 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=30 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=31 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=32 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=33 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=34 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=35 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=36 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=37 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=38 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=39 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=4 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=40 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=41 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=42 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=43 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=44 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=45 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=46 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=47 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=48 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=49 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=5 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=50 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=51 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=52 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=53 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=54 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=55 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=56 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=57 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=58 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=59 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=6 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=60 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=61 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=62 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=63 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=64 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=65 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=66 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=67 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=68 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=69 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=7 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=70 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=71 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=72 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=73 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=74 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=75 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=76 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=77 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=78 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=79 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=8 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=80 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=82 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=83 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=84 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=85 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=86 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=87 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=88 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=89 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=9 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=90 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=91 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=92 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=93 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=94 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=95 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=96 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=97 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=98 +POSTHOOK: Input: default@orcfile_merge1@ds=1/part=99 +#### A masked pattern was here #### +59521204047 +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) + FROM orcfile_merge1b WHERE ds='1' +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@orcfile_merge1b +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=0 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=1 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=10 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=11 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=12 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=13 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=14 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=15 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=16 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=17 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=18 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=19 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=2 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=20 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=21 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=22 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=23 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=24 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=25 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=26 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=27 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=28 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=29 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=3 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=30 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=31 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=32 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=33 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=34 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=35 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=36 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=37 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=38 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=39 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=4 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=40 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=41 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=42 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=43 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=44 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=45 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=46 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=47 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=48 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=49 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=5 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=50 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=51 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=52 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=53 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=54 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=55 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=56 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=57 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=58 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=59 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=6 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=60 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=61 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=62 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=63 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=64 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=65 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=66 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=67 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=68 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=69 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=7 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=70 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=71 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=72 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=73 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=74 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=75 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=76 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=77 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=78 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=79 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=8 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=80 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=82 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=83 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=84 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=85 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=86 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=87 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=88 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=89 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=9 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=90 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=91 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=92 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=93 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=94 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=95 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=96 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=97 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=98 +PREHOOK: Input: default@orcfile_merge1b@ds=1/part=99 +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) + FROM orcfile_merge1b WHERE ds='1' +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orcfile_merge1b +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=0 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=1 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=10 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=11 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=12 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=13 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=14 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=15 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=16 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=17 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=18 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=19 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=2 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=20 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=21 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=22 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=23 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=24 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=25 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=26 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=27 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=28 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=29 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=3 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=30 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=31 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=32 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=33 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=34 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=35 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=36 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=37 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=38 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=39 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=4 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=40 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=41 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=42 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=43 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=44 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=45 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=46 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=47 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=48 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=49 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=5 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=50 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=51 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=52 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=53 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=54 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=55 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=56 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=57 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=58 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=59 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=6 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=60 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=61 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=62 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=63 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=64 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=65 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=66 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=67 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=68 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=69 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=7 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=70 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=71 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=72 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=73 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=74 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=75 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=76 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=77 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=78 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=79 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=8 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=80 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=82 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=83 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=84 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=85 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=86 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=87 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=88 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=89 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=9 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=90 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=91 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=92 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=93 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=94 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=95 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=96 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=97 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=98 +POSTHOOK: Input: default@orcfile_merge1b@ds=1/part=99 +#### A masked pattern was here #### +59521204047 +PREHOOK: query: DROP TABLE orcfile_merge1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@orcfile_merge1 +PREHOOK: Output: default@orcfile_merge1 +POSTHOOK: query: DROP TABLE orcfile_merge1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@orcfile_merge1 +POSTHOOK: Output: default@orcfile_merge1 +PREHOOK: query: DROP TABLE orcfile_merge1b +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@orcfile_merge1b +PREHOOK: Output: default@orcfile_merge1b +POSTHOOK: query: DROP TABLE orcfile_merge1b +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@orcfile_merge1b +POSTHOOK: Output: default@orcfile_merge1b diff --git ql/src/test/results/clientpositive/tez/orc_merge2.q.out ql/src/test/results/clientpositive/tez/orc_merge2.q.out new file mode 100644 index 0000000..d713276 --- /dev/null +++ ql/src/test/results/clientpositive/tez/orc_merge2.q.out @@ -0,0 +1,282 @@ +PREHOOK: query: DROP TABLE orcfile_merge2a +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orcfile_merge2a +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE orcfile_merge2a (key INT, value STRING) + PARTITIONED BY (one string, two string, three string) + STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE orcfile_merge2a (key INT, value STRING) + PARTITIONED BY (one string, two string, three string) + STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcfile_merge2a +PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge2a PARTITION (one='1', two, three) + SELECT key, value, PMOD(HASH(key), 10) as two, + PMOD(HASH(value), 10) as three + FROM src +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge2a PARTITION (one='1', two, three) + SELECT key, value, PMOD(HASH(key), 10) as two, + PMOD(HASH(value), 10) as three + FROM src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 10) (type: int), (hash(value) pmod 10) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int), _col3 (type: int) + sort order: ++ + Map-reduce partition columns: _col2 (type: int), _col3 (type: int) + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int) + Reducer 2 + Reduce Operator Tree: + Extract + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge2a + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + one 1 + three + two + replace: true + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge2a + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge2a PARTITION (one='1', two, three) + SELECT key, value, PMOD(HASH(key), 10) as two, + PMOD(HASH(value), 10) as three + FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@orcfile_merge2a@one=1 +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge2a PARTITION (one='1', two, three) + SELECT key, value, PMOD(HASH(key), 10) as two, + PMOD(HASH(value), 10) as three + FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=0/three=2 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=0/three=8 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=1/three=3 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=1/three=9 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=2/three=0 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=2/three=4 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=3/three=1 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=3/three=5 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=4/three=2 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=4/three=6 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=5/three=3 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=5/three=7 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=6/three=4 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=6/three=8 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=7/three=5 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=7/three=9 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=8/three=0 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=8/three=6 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=9/three=1 +POSTHOOK: Output: default@orcfile_merge2a@one=1/two=9/three=7 +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=0,three=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=0,three=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=0,three=8).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=0,three=8).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=1,three=3).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=1,three=3).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=1,three=9).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=1,three=9).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=2,three=0).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=2,three=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=2,three=4).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=2,three=4).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=3,three=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=3,three=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=3,three=5).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=3,three=5).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=4,three=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=4,three=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=4,three=6).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=4,three=6).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=5,three=3).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=5,three=3).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=5,three=7).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=5,three=7).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=6,three=4).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=6,three=4).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=6,three=8).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=6,three=8).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=7,three=5).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=7,three=5).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=7,three=9).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=7,three=9).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=8,three=0).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=8,three=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=8,three=6).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=8,three=6).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=9,three=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=9,three=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=9,three=7).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=9,three=7).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: DESC FORMATTED orcfile_merge2a partition (one='1', two='0', three='2') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@orcfile_merge2a +POSTHOOK: query: DESC FORMATTED orcfile_merge2a partition (one='1', two='0', three='2') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@orcfile_merge2a +# col_name data_type comment + +key int +value string + +# Partition Information +# col_name data_type comment + +one string +two string +three string + +# Detailed Partition Information +Partition Value: [1, 0, 2] +Database: default +Table: orcfile_merge2a +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 4 + rawDataSize 376 + totalSize 320 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) + FROM orcfile_merge2a +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@orcfile_merge2a +PREHOOK: Input: default@orcfile_merge2a@one=1/two=0/three=2 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=0/three=8 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=1/three=3 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=1/three=9 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=2/three=0 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=2/three=4 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=3/three=1 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=3/three=5 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=4/three=2 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=4/three=6 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=5/three=3 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=5/three=7 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=6/three=4 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=6/three=8 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=7/three=5 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=7/three=9 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=8/three=0 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=8/three=6 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=9/three=1 +PREHOOK: Input: default@orcfile_merge2a@one=1/two=9/three=7 +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) + FROM orcfile_merge2a +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orcfile_merge2a +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=0/three=2 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=0/three=8 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=1/three=3 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=1/three=9 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=2/three=0 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=2/three=4 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=3/three=1 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=3/three=5 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=4/three=2 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=4/three=6 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=5/three=3 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=5/three=7 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=6/three=4 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=6/three=8 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=7/three=5 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=7/three=9 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=8/three=0 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=8/three=6 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=9/three=1 +POSTHOOK: Input: default@orcfile_merge2a@one=1/two=9/three=7 +#### A masked pattern was here #### +-4209012844 +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value, '1', PMOD(HASH(key), 10), + PMOD(HASH(value), 10)) USING 'tr \t _' AS (c) + FROM src +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value, '1', PMOD(HASH(key), 10), + PMOD(HASH(value), 10)) USING 'tr \t _' AS (c) + FROM src +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +-4209012844 +PREHOOK: query: DROP TABLE orcfile_merge2a +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@orcfile_merge2a +PREHOOK: Output: default@orcfile_merge2a +POSTHOOK: query: DROP TABLE orcfile_merge2a +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@orcfile_merge2a +POSTHOOK: Output: default@orcfile_merge2a diff --git ql/src/test/results/clientpositive/tez/orc_merge3.q.out ql/src/test/results/clientpositive/tez/orc_merge3.q.out new file mode 100644 index 0000000..38c65fa --- /dev/null +++ ql/src/test/results/clientpositive/tez/orc_merge3.q.out @@ -0,0 +1,201 @@ +PREHOOK: query: DROP TABLE orcfile_merge3a +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orcfile_merge3a +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE orcfile_merge3b +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orcfile_merge3b +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE orcfile_merge3a (key int, value string) + PARTITIONED BY (ds string) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE orcfile_merge3a (key int, value string) + PARTITIONED BY (ds string) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcfile_merge3a +PREHOOK: query: CREATE TABLE orcfile_merge3b (key int, value string) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE orcfile_merge3b (key int, value string) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcfile_merge3b +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='1') + SELECT * FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@orcfile_merge3a@ds=1 +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='1') + SELECT * FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@orcfile_merge3a@ds=1 +POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='2') + SELECT * FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@orcfile_merge3a@ds=2 +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='2') + SELECT * FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@orcfile_merge3a@ds=2 +POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge3b + SELECT key, value FROM orcfile_merge3a +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge3b + SELECT key, value FROM orcfile_merge3a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: orcfile_merge3a + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge3b + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge3b + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3b + SELECT key, value FROM orcfile_merge3a +PREHOOK: type: QUERY +PREHOOK: Input: default@orcfile_merge3a +PREHOOK: Input: default@orcfile_merge3a@ds=1 +PREHOOK: Input: default@orcfile_merge3a@ds=2 +PREHOOK: Output: default@orcfile_merge3b +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3b + SELECT key, value FROM orcfile_merge3a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orcfile_merge3a +POSTHOOK: Input: default@orcfile_merge3a@ds=1 +POSTHOOK: Input: default@orcfile_merge3a@ds=2 +POSTHOOK: Output: default@orcfile_merge3b +POSTHOOK: Lineage: orcfile_merge3b.key SIMPLE [(orcfile_merge3a)orcfile_merge3a.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: orcfile_merge3b.value SIMPLE [(orcfile_merge3a)orcfile_merge3a.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: DESC FORMATTED orcfile_merge3b +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@orcfile_merge3b +POSTHOOK: query: DESC FORMATTED orcfile_merge3b +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@orcfile_merge3b +# col_name data_type comment + +key int +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 1000 + rawDataSize 94000 + totalSize 2502 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) + FROM orcfile_merge3a +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@orcfile_merge3a +PREHOOK: Input: default@orcfile_merge3a@ds=1 +PREHOOK: Input: default@orcfile_merge3a@ds=2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) + FROM orcfile_merge3a +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orcfile_merge3a +POSTHOOK: Input: default@orcfile_merge3a@ds=1 +POSTHOOK: Input: default@orcfile_merge3a@ds=2 +#### A masked pattern was here #### +14412220296 +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) + FROM orcfile_merge3b +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@orcfile_merge3b +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) + FROM orcfile_merge3b +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orcfile_merge3b +#### A masked pattern was here #### +14412220296 +PREHOOK: query: DROP TABLE orcfile_merge3a +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@orcfile_merge3a +PREHOOK: Output: default@orcfile_merge3a +POSTHOOK: query: DROP TABLE orcfile_merge3a +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@orcfile_merge3a +POSTHOOK: Output: default@orcfile_merge3a +PREHOOK: query: DROP TABLE orcfile_merge3b +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@orcfile_merge3b +PREHOOK: Output: default@orcfile_merge3b +POSTHOOK: query: DROP TABLE orcfile_merge3b +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@orcfile_merge3b +POSTHOOK: Output: default@orcfile_merge3b diff --git ql/src/test/results/clientpositive/tez/orc_merge4.q.out ql/src/test/results/clientpositive/tez/orc_merge4.q.out new file mode 100644 index 0000000..178760b --- /dev/null +++ ql/src/test/results/clientpositive/tez/orc_merge4.q.out @@ -0,0 +1,300 @@ +PREHOOK: query: DROP TABLE orcfile_merge3a +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orcfile_merge3a +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE orcfile_merge3b +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orcfile_merge3b +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE orcfile_merge3a (key int, value string) + PARTITIONED BY (ds string) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE orcfile_merge3a (key int, value string) + PARTITIONED BY (ds string) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcfile_merge3a +PREHOOK: query: CREATE TABLE orcfile_merge3b (key int, value string) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE orcfile_merge3b (key int, value string) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcfile_merge3b +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='1') + SELECT * FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@orcfile_merge3a@ds=1 +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='1') + SELECT * FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@orcfile_merge3a@ds=1 +POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: DESC FORMATTED orcfile_merge3a PARTITION (ds='1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@orcfile_merge3a +POSTHOOK: query: DESC FORMATTED orcfile_merge3a PARTITION (ds='1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@orcfile_merge3a +# col_name data_type comment + +key int +value string + +# Partition Information +# col_name data_type comment + +ds string + +# Detailed Partition Information +Partition Value: [1] +Database: default +Table: orcfile_merge3a +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 47000 + totalSize 2460 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='1') + SELECT * FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@orcfile_merge3a@ds=1 +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='1') + SELECT * FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@orcfile_merge3a@ds=1 +POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='2') + SELECT * FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@orcfile_merge3a@ds=2 +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='2') + SELECT * FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@orcfile_merge3a@ds=2 +POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: DESC FORMATTED orcfile_merge3a PARTITION (ds='1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@orcfile_merge3a +POSTHOOK: query: DESC FORMATTED orcfile_merge3a PARTITION (ds='1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@orcfile_merge3a +# col_name data_type comment + +key int +value string + +# Partition Information +# col_name data_type comment + +ds string + +# Detailed Partition Information +Partition Value: [1] +Database: default +Table: orcfile_merge3a +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 47000 + totalSize 2460 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: DESC FORMATTED orcfile_merge3a PARTITION (ds='2') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@orcfile_merge3a +POSTHOOK: query: DESC FORMATTED orcfile_merge3a PARTITION (ds='2') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@orcfile_merge3a +# col_name data_type comment + +key int +value string + +# Partition Information +# col_name data_type comment + +ds string + +# Detailed Partition Information +Partition Value: [2] +Database: default +Table: orcfile_merge3a +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 47000 + totalSize 2460 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge3b + SELECT key, value FROM orcfile_merge3a +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge3b + SELECT key, value FROM orcfile_merge3a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: orcfile_merge3a + Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.orcfile_merge3b + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.orcfile_merge3b + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3b + SELECT key, value FROM orcfile_merge3a +PREHOOK: type: QUERY +PREHOOK: Input: default@orcfile_merge3a +PREHOOK: Input: default@orcfile_merge3a@ds=1 +PREHOOK: Input: default@orcfile_merge3a@ds=2 +PREHOOK: Output: default@orcfile_merge3b +POSTHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3b + SELECT key, value FROM orcfile_merge3a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orcfile_merge3a +POSTHOOK: Input: default@orcfile_merge3a@ds=1 +POSTHOOK: Input: default@orcfile_merge3a@ds=2 +POSTHOOK: Output: default@orcfile_merge3b +POSTHOOK: Lineage: orcfile_merge3b.key SIMPLE [(orcfile_merge3a)orcfile_merge3a.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: orcfile_merge3b.value SIMPLE [(orcfile_merge3a)orcfile_merge3a.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) + FROM orcfile_merge3a +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@orcfile_merge3a +PREHOOK: Input: default@orcfile_merge3a@ds=1 +PREHOOK: Input: default@orcfile_merge3a@ds=2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) + FROM orcfile_merge3a +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orcfile_merge3a +POSTHOOK: Input: default@orcfile_merge3a@ds=1 +POSTHOOK: Input: default@orcfile_merge3a@ds=2 +#### A masked pattern was here #### +14412220296 +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) + FROM orcfile_merge3b +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@orcfile_merge3b +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) + FROM orcfile_merge3b +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orcfile_merge3b +#### A masked pattern was here #### +14412220296 +PREHOOK: query: DROP TABLE orcfile_merge3a +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@orcfile_merge3a +PREHOOK: Output: default@orcfile_merge3a +POSTHOOK: query: DROP TABLE orcfile_merge3a +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@orcfile_merge3a +POSTHOOK: Output: default@orcfile_merge3a +PREHOOK: query: DROP TABLE orcfile_merge3b +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@orcfile_merge3b +PREHOOK: Output: default@orcfile_merge3b +POSTHOOK: query: DROP TABLE orcfile_merge3b +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@orcfile_merge3b +POSTHOOK: Output: default@orcfile_merge3b