diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java index a164b1245c..604e013c0f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java @@ -51,6 +51,7 @@ import org.apache.thrift.TException; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.hive.metastore.api.FieldSchema; /** * Verify that the information in the metastore matches what is on the @@ -278,7 +279,11 @@ void checkTable(Table table, List parts, pr.setTableName(partition.getTable().getTableName()); result.getPartitionsNotOnFs().add(pr); } - + if(!isExistPartitionPathValid(fs,partPath)){ + String message="invalid subdirs in datalocation:"+partPath.toString(); + LOG.error(message); + throw new HiveException(message); + } for (int i = 0; i < partition.getSpec().size(); i++) { partPaths.add(partPath.makeQualified(fs)); partPath = partPath.getParent(); @@ -290,6 +295,19 @@ void checkTable(Table table, List parts, } } + boolean isExistPartitionPathValid(FileSystem fs,Path partPath)throws IOException{ + FileStatus[] files = fs.listStatus(partPath); + if(files==null||files.length==0){ + return true; + } + for(FileStatus fss:files){ + if(fss.isDirectory()){ + return false; + } + } + return true; + } + /** * Find partitions on the fs that are unknown to the metastore. * @@ -319,6 +337,11 @@ void findUnknownPartitions(Table table, Set partPaths, // we should now only have the unexpected folders left for (Path partPath : allPartDirs) { + if(!isVaildPartitionPath(table,partPath)){ + String message="invalid data path:"+partPath.toString(); + LOG.error(message); + throw new HiveException(message); + } FileSystem fs = partPath.getFileSystem(conf); String partitionName = getPartitionName(fs.makeQualified(tablePath), partPath); @@ -333,6 +356,35 @@ void findUnknownPartitions(Table table, Set partPaths, } } + boolean isVaildPartitionPath(Table table,Path partpath){ + Path tablePath = table.getPath(); + String partpathinfo=partpath.toString(); + String partinfo=partpathinfo.substring(tablePath.toString().length()+1,partpathinfo.length()); + if(partinfo==null||"".equals(partinfo)){ + return false; + } + String[] parts=partinfo.split("/"); + if(parts==null||parts.length==0){ + return false; + } + Map partsmap=new java.util.HashMap(); + for(String part:parts){ + int index=part.indexOf("="); + if(index<0){ + continue; + } + String partname=part.substring(0,index); + partsmap.put(partname,partname); + } + for (FieldSchema field : table.getPartCols()) { + String val = partsmap.get(field.getName()); + if (val == null || val.isEmpty()) { + return false; + } + } + return true; + } + /** * Get the partition name from the path. *