diff --git shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java index dd5be74..3902cce 100644 --- shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java +++ shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java @@ -28,7 +28,10 @@ import java.net.URL; import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import javax.security.auth.Subject; import javax.security.auth.login.LoginException; @@ -160,10 +163,17 @@ public class Hadoop20Shims implements HadoopShims { } public InputSplitShim(CombineFileSplit old) throws IOException { - super(old); + super(old.getJob(), old.getPaths(), old.getStartOffsets(), + old.getLengths(), dedup(old.getLocations())); _isShrinked = false; } + private static String[] dedup(String[] locations) { + Set dedup = new HashSet(); + Collections.addAll(dedup, locations); + return dedup.toArray(new String[dedup.size()]); + } + @Override public void shrinkSplit(long length) { _isShrinked = true; diff --git shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java index 5832f67..e35c7ce 100644 --- shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java +++ shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java @@ -25,7 +25,10 @@ import java.net.URI; import java.net.URISyntaxException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -160,10 +163,17 @@ public abstract class HadoopShimsSecure implements HadoopShims { } public InputSplitShim(CombineFileSplit old) throws IOException { - super(old); + super(old.getJob(), old.getPaths(), old.getStartOffsets(), + old.getLengths(), dedup(old.getLocations())); _isShrinked = false; } + private static String[] dedup(String[] locations) { + Set dedup = new HashSet(); + Collections.addAll(dedup, locations); + return dedup.toArray(new String[dedup.size()]); + } + @Override public void shrinkSplit(long length) { _isShrinked = true;