diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFPercentile.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFPercentile.java index 4186a9d..1757ed4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFPercentile.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFPercentile.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.exec.UDAF; import org.apache.hadoop.hive.ql.exec.UDAFEvaluator; import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.LongWritable; /** @@ -44,6 +45,12 @@ + "pc can be a double or double array") public class UDAFPercentile extends UDAF { + private static final Comparator COMPARATOR; + + static { + COMPARATOR = ShimLoader.getHadoopShims().getLongComparator(); + } + /** * A state class to store intermediate aggregation results. */ @@ -59,7 +66,7 @@ @Override public int compare(Map.Entry o1, Map.Entry o2) { - return o1.getKey().compareTo(o2.getKey()); + return COMPARATOR.compare(o1.getKey(), o2.getKey()); } } diff --git ql/src/test/queries/clientpositive/udaf_percentile.q ql/src/test/queries/clientpositive/udaf_percentile.q new file mode 100644 index 0000000..8ebf01d --- /dev/null +++ ql/src/test/queries/clientpositive/udaf_percentile.q @@ -0,0 +1 @@ +select percentile(cast(key as bigint), 0.3) from src; diff --git ql/src/test/results/clientpositive/udaf_percentile.q.out ql/src/test/results/clientpositive/udaf_percentile.q.out new file mode 100644 index 0000000..1861a9f --- /dev/null +++ ql/src/test/results/clientpositive/udaf_percentile.q.out @@ -0,0 +1,9 @@ +PREHOOK: query: select percentile(cast(key as bigint), 0.3) from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select percentile(cast(key as bigint), 0.3) from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +169.0 diff --git shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java index 6ff1a84..f03790b 100644 --- shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java +++ shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java @@ -29,6 +29,7 @@ import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -47,6 +48,7 @@ import org.apache.hadoop.fs.Trash; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil; +import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.ClusterStatus; import org.apache.hadoop.mapred.FileInputFormat; @@ -160,6 +162,16 @@ public void setTotalOrderPartitionFile(JobConf jobConf, Path partitionFile){ TotalOrderPartitioner.setPartitionFile(jobConf, partitionFile); } + @Override + public Comparator getLongComparator() { + return new Comparator() { + @Override + public int compare(LongWritable o1, LongWritable o2) { + return o1.compareTo(o2); + } + }; + } + public static class InputSplitShim extends CombineFileSplit implements HadoopShims.InputSplitShim { long shrinkedLength; boolean _isShrinked; diff --git shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java index 62f7984..f2f3dab 100644 --- shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java +++ shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java @@ -21,6 +21,7 @@ import java.net.InetSocketAddress; import java.net.MalformedURLException; import java.net.URL; +import java.util.Comparator; import java.util.Iterator; import java.net.URI; @@ -34,6 +35,7 @@ import org.apache.hadoop.fs.ProxyFileSystem; import org.apache.hadoop.fs.Trash; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapred.JobTracker; import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.mapred.ClusterStatus; @@ -150,6 +152,16 @@ public void setTotalOrderPartitionFile(JobConf jobConf, Path partitionFile){ TotalOrderPartitioner.setPartitionFile(jobConf, partitionFile); } + @Override + public Comparator getLongComparator() { + return new Comparator() { + @Override + public int compare(LongWritable o1, LongWritable o2) { + return o1.compareTo(o2); + } + }; + } + /** * Returns a shim to wrap MiniMrCluster */ diff --git shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index cac1594..47591d2 100644 --- shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -22,6 +22,7 @@ import java.net.InetSocketAddress; import java.net.MalformedURLException; import java.net.URL; +import java.util.Comparator; import java.util.Iterator; import java.util.Map; import java.net.URI; @@ -39,6 +40,7 @@ import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.Trash; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapred.ClusterStatus; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.MiniMRCluster; @@ -166,6 +168,16 @@ public void setTotalOrderPartitionFile(JobConf jobConf, Path partitionFile){ TotalOrderPartitioner.setPartitionFile(jobConf, partitionFile); } + @Override + public Comparator getLongComparator() { + return new Comparator() { + @Override + public int compare(LongWritable o1, LongWritable o2) { + return o1.compareTo(o2); + } + }; + } + /** * Returns a shim to wrap MiniMrCluster */ diff --git shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java index 0d5615c..0e982ee 100644 --- shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java +++ shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java @@ -25,6 +25,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.security.PrivilegedExceptionAction; +import java.util.Comparator; import java.util.Iterator; import java.util.List; @@ -38,17 +39,14 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; -import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapred.ClusterStatus; -import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobProfile; import org.apache.hadoop.mapred.JobStatus; import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.mapred.RunningJob; -import org.apache.hadoop.mapred.TaskCompletionEvent; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.JobID; @@ -354,6 +352,9 @@ public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration co * @param partition */ void setTotalOrderPartitionFile(JobConf jobConf, Path partition); + + Comparator getLongComparator(); + /** * InputSplitShim. *