From 45b525db3c6dac251cb9d3b13c5271e218ec9ff6 Mon Sep 17 00:00:00 2001 From: Sammy Yu Date: Wed, 1 Sep 2010 23:03:06 -0700 Subject: [PATCH 2/2] HIVE-1610. Added additional schema check to doGetPartitionDescFromPath --- .../hadoop/hive/ql/io/HiveFileFormatUtils.java | 20 +++++++++++++++++++- .../hadoop/hive/ql/io/TestHiveFileFormatUtils.java | 17 +++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java index 1845948..90391e0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.io; import java.io.File; import java.io.IOException; +import java.net.URI; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; @@ -301,7 +302,7 @@ public final class HiveFileFormatUtils { // LOG.warn("exact match not found, try ripping input path's theme and authority"); part = pathToPartitionInfo.get(dirPath); } - + if (part == null) { String dirStr = dir.toString(); int dirPathIndex = dirPath.lastIndexOf(File.separator); @@ -322,6 +323,23 @@ public final class HiveFileFormatUtils { dirStrIndex = dirStr.lastIndexOf(File.separator); } } + + // try to match the schema to deal with ports in URI + if (part == null) { + for (Map.Entry entry: pathToPartitionInfo.entrySet()) { + PartitionDesc partDesc = entry.getValue(); + Path partPath = new Path(entry.getKey()); + URI partURI = partPath.toUri(); + String partScheme = partURI.getScheme(); + if (partScheme != null && partScheme.equals(dir.toUri().getScheme())) { + if (dir.toUri().getPath().startsWith(partURI.getPath())) { + part = partDesc; + break; + } + } + } + } + return part; } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/TestHiveFileFormatUtils.java b/ql/src/test/org/apache/hadoop/hive/ql/io/TestHiveFileFormatUtils.java index efc18ee..09e0585 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/TestHiveFileFormatUtils.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/TestHiveFileFormatUtils.java @@ -130,4 +130,21 @@ public class TestHiveFileFormatUtils extends TestCase { } + public void testGetPartitionDescFromPathWithPort() throws IOException { + + PartitionDesc partDesc_1 = new PartitionDesc(); + + Map pathToPartitionInfo = new HashMap(); + Map emptyPathToPartitionInfo = new HashMap(); + + pathToPartitionInfo.put( + new Path("hdfs://mycluster:8020/part1/-mr-10002").toString(), partDesc_1); + + PartitionDesc ret = null; + + ret = HiveFileFormatUtils.getPartitionDescFromPathRecursively( + pathToPartitionInfo, new Path("hdfs://mycluster/part1/-mr-10002/000000_0"), + IOPrepareCache.get().allocatePartitionDescMap()); + assertEquals("file:///tbl/part1 not found.", partDesc_1, ret); + } } -- 1.6.5.2