diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml index 187b96e..7dfbd85 100644 --- a/itests/qtest/pom.xml +++ b/itests/qtest/pom.xml @@ -225,6 +225,12 @@ org.apache.hadoop + hadoop-distcp + ${hadoop-23.version} + test + + + org.apache.hadoop hadoop-hdfs ${hadoop-23.version} tests diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index ba9f153..7c62ba5 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -25,6 +25,7 @@ minimr.query.files=auto_sortmerge_join_16.q,\ infer_bucket_sort_num_buckets.q,\ infer_bucket_sort_reducers_power_two.q,\ input16_cc.q,\ + insert_dir_distcp.q,\ join1.q,\ leftsemijoin_mr.q,\ list_bucket_dml_10.q,\ @@ -330,6 +331,7 @@ minitez.query.files=bucket_map_join_tez1.q,\ hybridgrace_hashjoin_1.q,\ hybridgrace_hashjoin_2.q,\ mapjoin_decimal.q,\ + insert_dir_distcp.q,\ lvj_mapjoin.q, \ mrr.q,\ orc_ppd_basic.q,\ diff --git a/ql/src/test/queries/clientpositive/insert_dir_distcp.q b/ql/src/test/queries/clientpositive/insert_dir_distcp.q new file mode 100644 index 0000000..6582938 --- /dev/null +++ b/ql/src/test/queries/clientpositive/insert_dir_distcp.q @@ -0,0 +1,9 @@ +set hive.exec.copyfile.maxsize=400; + +set tez.am.log.level=INFO; +set tez.task.log.level=INFO; +-- see TEZ-2931 for using INFO logging + +insert overwrite directory '/tmp/src' select * from src; + +dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/src/; diff --git a/ql/src/test/results/clientpositive/insert_dir_distcp.q.out b/ql/src/test/results/clientpositive/insert_dir_distcp.q.out new file mode 100644 index 0000000..b70fa01 --- /dev/null +++ b/ql/src/test/results/clientpositive/insert_dir_distcp.q.out @@ -0,0 +1,14 @@ +PREHOOK: query: -- see TEZ-2931 for using INFO logging + +#### A masked pattern was here #### +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: -- see TEZ-2931 for using INFO logging + +#### A masked pattern was here #### +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +Found 1 items +#### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/tez/insert_dir_distcp.q.out b/ql/src/test/results/clientpositive/tez/insert_dir_distcp.q.out new file mode 100644 index 0000000..b70fa01 --- /dev/null +++ b/ql/src/test/results/clientpositive/tez/insert_dir_distcp.q.out @@ -0,0 +1,14 @@ +PREHOOK: query: -- see TEZ-2931 for using INFO logging + +#### A masked pattern was here #### +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: -- see TEZ-2931 for using INFO logging + +#### A masked pattern was here #### +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +Found 1 items +#### A masked pattern was here #### diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index 9e9973b..0d2c802 100644 --- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -1172,11 +1172,14 @@ public boolean runDistCp(Path src, Path dst, Configuration conf) throws IOExcept options.preserve(FileAttribute.BLOCKSIZE); try { + conf.setBoolean("mapred.mapper.new-api", true); DistCp distcp = new DistCp(conf, options); distcp.execute(); return true; } catch (Exception e) { throw new IOException("Cannot execute DistCp process: " + e, e); + } finally { + conf.setBoolean("mapred.mapper.new-api", false); } }