diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java index 9ec22ca..3dd0f6f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.ErrorMsg; @@ -62,10 +63,22 @@ public LoadSemanticAnalyzer(HiveConf conf) throws SemanticException { public static FileStatus[] matchFilesOrDir(FileSystem fs, Path path) throws IOException { - FileStatus[] srcs = fs.globStatus(path); + FileStatus[] srcs = fs.globStatus(path, new PathFilter() { + @Override + public boolean accept(Path p) { + String name = p.getName(); + return name.equals("_metadata") ? true : !name.startsWith("_") && !name.startsWith("."); + } + }); if ((srcs != null) && srcs.length == 1) { if (srcs[0].isDir()) { - srcs = fs.listStatus(srcs[0].getPath()); + srcs = fs.listStatus(srcs[0].getPath(), new PathFilter() { + @Override + public boolean accept(Path p) { + String name = p.getName(); + return !name.startsWith("_") && !name.startsWith("."); + } + }); } } return (srcs); diff --git ql/src/test/queries/clientpositive/exim_hidden_files.q ql/src/test/queries/clientpositive/exim_hidden_files.q new file mode 100644 index 0000000..f58c9f9 --- /dev/null +++ ql/src/test/queries/clientpositive/exim_hidden_files.q @@ -0,0 +1,22 @@ +set hive.test.mode=true; +set hive.test.mode.prefix=; +set hive.test.mode.nosamplelist=exim_department,exim_employee; + +create table exim_employee ( emp_id int) partitioned by (emp_country string); +load data local inpath "../../data/files/test.dat" into table exim_employee partition (emp_country="in"); + +dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/exim_employee/emp_country=in/_logs; +dfs -touchz ${system:test.warehouse.dir}/exim_employee/emp_country=in/_logs/job.xml; +export table exim_employee to 'ql/test/data/exports/exim_employee'; +drop table exim_employee; + +create database importer; +use importer; + +import from 'ql/test/data/exports/exim_employee'; +describe formatted exim_employee; +select * from exim_employee; +dfs -rmr target/tmp/ql/test/data/exports/exim_employee; +drop table exim_employee; +drop database importer; +use default; diff --git ql/src/test/results/clientpositive/exim_hidden_files.q.out ql/src/test/results/clientpositive/exim_hidden_files.q.out new file mode 100644 index 0000000..7513262 --- /dev/null +++ ql/src/test/results/clientpositive/exim_hidden_files.q.out @@ -0,0 +1,116 @@ +PREHOOK: query: create table exim_employee ( emp_id int) partitioned by (emp_country string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: create table exim_employee ( emp_id int) partitioned by (emp_country string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@exim_employee +PREHOOK: query: load data local inpath "../../data/files/test.dat" into table exim_employee partition (emp_country="in") +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@exim_employee +POSTHOOK: query: load data local inpath "../../data/files/test.dat" into table exim_employee partition (emp_country="in") +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@exim_employee +POSTHOOK: Output: default@exim_employee@emp_country=in +PREHOOK: query: export table exim_employee to 'ql/test/data/exports/exim_employee' +PREHOOK: type: EXPORT +PREHOOK: Input: default@exim_employee@emp_country=in +#### A masked pattern was here #### +POSTHOOK: query: export table exim_employee to 'ql/test/data/exports/exim_employee' +POSTHOOK: type: EXPORT +POSTHOOK: Input: default@exim_employee@emp_country=in +#### A masked pattern was here #### +PREHOOK: query: drop table exim_employee +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@exim_employee +PREHOOK: Output: default@exim_employee +POSTHOOK: query: drop table exim_employee +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@exim_employee +POSTHOOK: Output: default@exim_employee +PREHOOK: query: create database importer +PREHOOK: type: CREATEDATABASE +POSTHOOK: query: create database importer +POSTHOOK: type: CREATEDATABASE +PREHOOK: query: use importer +PREHOOK: type: SWITCHDATABASE +POSTHOOK: query: use importer +POSTHOOK: type: SWITCHDATABASE +PREHOOK: query: import from 'ql/test/data/exports/exim_employee' +PREHOOK: type: IMPORT +POSTHOOK: query: import from 'ql/test/data/exports/exim_employee' +POSTHOOK: type: IMPORT +POSTHOOK: Output: importer@exim_employee +POSTHOOK: Output: importer@exim_employee@emp_country=in +PREHOOK: query: describe formatted exim_employee +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe formatted exim_employee +POSTHOOK: type: DESCTABLE +# col_name data_type comment + +emp_id int None + +# Partition Information +# col_name data_type comment + +emp_country string None + +# Detailed Table Information +Database: importer +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select * from exim_employee +PREHOOK: type: QUERY +PREHOOK: Input: importer@exim_employee +PREHOOK: Input: importer@exim_employee@emp_country=in +#### A masked pattern was here #### +POSTHOOK: query: select * from exim_employee +POSTHOOK: type: QUERY +POSTHOOK: Input: importer@exim_employee +POSTHOOK: Input: importer@exim_employee@emp_country=in +#### A masked pattern was here #### +1 in +2 in +3 in +4 in +5 in +6 in +#### A masked pattern was here #### +PREHOOK: query: drop table exim_employee +PREHOOK: type: DROPTABLE +PREHOOK: Input: importer@exim_employee +PREHOOK: Output: importer@exim_employee +POSTHOOK: query: drop table exim_employee +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: importer@exim_employee +POSTHOOK: Output: importer@exim_employee +PREHOOK: query: drop database importer +PREHOOK: type: DROPDATABASE +PREHOOK: Input: database:importer +PREHOOK: Output: database:importer +POSTHOOK: query: drop database importer +POSTHOOK: type: DROPDATABASE +POSTHOOK: Input: database:importer +POSTHOOK: Output: database:importer +PREHOOK: query: use default +PREHOOK: type: SWITCHDATABASE +POSTHOOK: query: use default +POSTHOOK: type: SWITCHDATABASE