diff --git build-common.xml build-common.xml
index 458d4ef..00c3680 100644
--- build-common.xml
+++ build-common.xml
@@ -426,7 +426,7 @@
-->
-
+
diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 2cdaeb6..6c72ec6 100644
--- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -349,6 +349,7 @@ public class HiveConf extends Configuration {
HIVEPARTITIONER("hive.mapred.partitioner", "org.apache.hadoop.hive.ql.io.DefaultHivePartitioner"),
HIVESCRIPTOPERATORTRUST("hive.exec.script.trust", false),
+ HIVEROWOFFSET("hive.exec.rowoffset", false),
HIVE_COMBINE_INPUT_FORMAT_SUPPORTS_SPLITTABLE("hive.hadoop.supports.splittable.combineinputformat", false),
diff --git conf/hive-default.xml conf/hive-default.xml
index 79ea477..9e756bc 100644
--- conf/hive-default.xml
+++ conf/hive-default.xml
@@ -460,6 +460,12 @@
+ hive.exec.rowoffset
+ false
+ Whether to provide the row offset virtual column
+
+
+
hive.task.progress
false
Whether Hive should periodically update task progress counters during execution. Enabling this allows task progress to be monitored more closely in the job tracker, but may impose a performance penalty. This flag is automatically set to true for jobs with hive.exec.dynamic.partition set to true.
diff --git lib/README lib/README
index 1c2f0b1..6cc2fba 100644
--- lib/README
+++ lib/README
@@ -13,3 +13,4 @@ Folowing is the list of external jars contained in this directory and the source
* commons-collections-3.2.1.jar - http://commons.apache.org/downloads/download_collections.cgi
* commons-lang-2.4.jar - http://commons.apache.org/downloads/download_lang.cgi
* json.jar - http://www.json.org/java/index.html
+* javaewah-0.2.jar - http://code.google.com/p/javaewah/downloads/
diff --git lib/javaewah-0.2.jar lib/javaewah-0.2.jar
new file mode 100644
index 0000000..1f48093
Binary files /dev/null and lib/javaewah-0.2.jar differ
diff --git lib/javaewah.LICENSE lib/javaewah.LICENSE
new file mode 100644
index 0000000..261eeb9
--- /dev/null
+++ lib/javaewah.LICENSE
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
index ba222f3..16a207e 100644
--- ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
@@ -122,6 +122,7 @@ import org.apache.hadoop.hive.ql.udf.UDFUpper;
import org.apache.hadoop.hive.ql.udf.UDFWeekOfYear;
import org.apache.hadoop.hive.ql.udf.UDFYear;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEWAHBitmap;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFBridge;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFCollectSet;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFContextNGrams;
@@ -146,6 +147,9 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFnGrams;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFArray;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFArrayContains;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFEWAHBitmapAnd;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFEWAHBitmapOr;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFEWAHBitmapEmpty;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCase;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCoalesce;
@@ -339,6 +343,10 @@ public final class FunctionRegistry {
registerGenericUDF("not", GenericUDFOPNot.class);
registerGenericUDF("!", GenericUDFOPNot.class);
+ registerGenericUDF("ewah_bitmap_and", GenericUDFEWAHBitmapAnd.class);
+ registerGenericUDF("ewah_bitmap_or", GenericUDFEWAHBitmapOr.class);
+ registerGenericUDF("ewah_bitmap_empty", GenericUDFEWAHBitmapEmpty.class);
+
// Aliases for Java Class Names
// These are used in getImplicitConvertUDFMethod
@@ -384,6 +392,8 @@ public final class FunctionRegistry {
registerGenericUDAF("ngrams", new GenericUDAFnGrams());
registerGenericUDAF("context_ngrams", new GenericUDAFContextNGrams());
+ registerGenericUDAF("ewah_bitmap", new GenericUDAFEWAHBitmap());
+
registerUDAF("percentile", UDAFPercentile.class);
diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
index ff74f08..96a1631 100644
--- ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
@@ -566,6 +566,17 @@ public class MapOperator extends Operator implements Serializable {
if (current != old.get()) {
old.set(current);
}
+ } else if (vc.equals(VirtualColumn.ROWOFFSET)) {
+ long current = ioCxt.getCurrentRow();
+ LongWritable old = (LongWritable) this.vcValues[i];
+ if (old == null) {
+ old = new LongWritable(current);
+ this.vcValues[i] = old;
+ continue;
+ }
+ if (current != old.get()) {
+ old.set(current);
+ }
}
}
}
diff --git ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndex.java ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndex.java
index 308d985..591c9ff 100644
--- ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndex.java
+++ ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndex.java
@@ -30,7 +30,9 @@ public class HiveIndex {
public static String INDEX_TABLE_CREATETIME = "hive.index.basetbl.dfs.lastModifiedTime";
public static enum IndexType {
- COMPACT_SUMMARY_TABLE("compact", "org.apache.hadoop.hive.ql.index.compact.CompactIndexHandler");
+ COMPACT_SUMMARY_TABLE("compact", "org.apache.hadoop.hive.ql.index.compact.CompactIndexHandler"),
+ BITMAP_TABLE("bitmap",
+"org.apache.hadoop.hive.ql.index.bitmap.BitmapIndexHandler");
private IndexType(String indexType, String className) {
indexTypeName = indexType;
diff --git ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexResult.java ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexResult.java
new file mode 100644
index 0000000..4eda17b
--- /dev/null
+++ ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexResult.java
@@ -0,0 +1,193 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.index;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.columnar.BytesRefWritable;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.LineRecordReader.LineReader;
+
+/**
+ * HiveIndexResult parses the input stream from an index query
+ * to generate a list of file splits to query.
+ */
+public class HiveIndexResult {
+
+ public static final Log l4j =
+ LogFactory.getLog(HiveIndexResult.class.getSimpleName());
+
+ // IndexBucket
+ static class IBucket {
+ private String name = null;
+ private SortedSet offsets = new TreeSet();
+
+ public IBucket(String n) {
+ name = n;
+ }
+
+ public void add(Long offset) {
+ offsets.add(offset);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public SortedSet getOffsets() {
+ return offsets;
+ }
+
+ public boolean equals(Object obj) {
+ if (obj.getClass() != this.getClass()) {
+ return false;
+ }
+ return (((IBucket) obj).name.compareToIgnoreCase(this.name) == 0);
+ }
+ }
+
+ JobConf job = null;
+ BytesRefWritable[] bytesRef = new BytesRefWritable[2];
+ boolean ignoreHdfsLoc = false;
+
+ public HiveIndexResult(String indexFile, JobConf conf) throws IOException,
+ HiveException {
+ job = conf;
+
+ bytesRef[0] = new BytesRefWritable();
+ bytesRef[1] = new BytesRefWritable();
+ ignoreHdfsLoc = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_INDEX_IGNORE_HDFS_LOC);
+
+ if (indexFile != null) {
+ Path indexFilePath = new Path(indexFile);
+ FileSystem fs = FileSystem.get(conf);
+ FileStatus indexStat = fs.getFileStatus(indexFilePath);
+ List paths = new ArrayList();
+ if (indexStat.isDir()) {
+ FileStatus[] fss = fs.listStatus(indexFilePath);
+ for (FileStatus f : fss) {
+ paths.add(f.getPath());
+ }
+ } else {
+ paths.add(indexFilePath);
+ }
+
+ for (Path indexFinalPath : paths) {
+ FSDataInputStream ifile = fs.open(indexFinalPath);
+ LineReader lr = new LineReader(ifile, conf);
+ try {
+ Text line = new Text();
+ while (lr.readLine(line) > 0) {
+ add(line);
+ }
+ }
+ finally {
+ // this will close the input stream
+ lr.close();
+ }
+ }
+ }
+ }
+
+ Map buckets = new HashMap();
+
+ private void add(Text line) throws HiveException {
+ String l = line.toString();
+ byte[] bytes = l.getBytes();
+ int firstEnd = 0;
+ int i = 0;
+ for (int index = 0; index < bytes.length; index++) {
+ if (bytes[index] == LazySimpleSerDe.DefaultSeparators[0]) {
+ i++;
+ firstEnd = index;
+ }
+ }
+ if (i > 1) {
+ throw new HiveException(
+ "Bad index file row (index file should only contain two columns: bucket_file_name and offset lists.) ."
+ + line.toString());
+ }
+ String bucketFileName = new String(bytes, 0, firstEnd);
+
+ if (ignoreHdfsLoc) {
+ Path tmpPath = new Path(bucketFileName);
+ bucketFileName = tmpPath.toUri().getPath();
+ }
+ IBucket bucket = buckets.get(bucketFileName);
+ if (bucket == null) {
+ bucket = new IBucket(bucketFileName);
+ buckets.put(bucketFileName, bucket);
+ }
+
+ int currentStart = firstEnd + 1;
+ int currentEnd = firstEnd + 1;
+ for (; currentEnd < bytes.length; currentEnd++) {
+ if (bytes[currentEnd] == LazySimpleSerDe.DefaultSeparators[1]) {
+ String one_offset = new String(bytes, currentStart, currentEnd
+ - currentStart);
+ Long offset = Long.parseLong(one_offset);
+ bucket.getOffsets().add(offset);
+ currentStart = currentEnd + 1;
+ }
+ }
+ String one_offset = new String(bytes, currentStart, currentEnd
+ - currentStart);
+ bucket.getOffsets().add(Long.parseLong(one_offset));
+ }
+
+ public boolean contains(FileSplit split) throws HiveException {
+
+ if (buckets == null) {
+ return false;
+ }
+ String bucketName = split.getPath().toString();
+ IBucket bucket = buckets.get(bucketName);
+ if (bucket == null) {
+ bucketName = split.getPath().toUri().getPath();
+ bucket = buckets.get(bucketName);
+ if (bucket == null) {
+ return false;
+ }
+ }
+
+ for (Long offset : bucket.getOffsets()) {
+ if ((offset >= split.getStart())
+ && (offset <= split.getStart() + split.getLength())) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
diff --git ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexedInputFormat.java ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexedInputFormat.java
new file mode 100644
index 0000000..a3d74e2
--- /dev/null
+++ ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexedInputFormat.java
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.index;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
+import org.apache.hadoop.hive.ql.io.HiveInputFormat;
+import org.apache.hadoop.hive.ql.io.IOPrepareCache;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.PartitionDesc;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileSplit;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+
+/**
+ * Input format for doing queries that use indexes.
+ * Uses a blockfilter file to specify the blocks to query.
+ */
+public class HiveIndexedInputFormat extends HiveInputFormat {
+ public static final Log l4j = LogFactory.getLog("HiveIndexInputFormat");
+ private final String indexFile;
+
+ public HiveIndexedInputFormat() {
+ super();
+ indexFile = "hive.index.blockfilter.file";
+ }
+
+ public HiveIndexedInputFormat(String indexFileName) {
+ indexFile = indexFileName;
+ }
+
+ public InputSplit[] doGetSplits(JobConf job, int numSplits) throws IOException {
+
+ super.init(job);
+
+ Path[] dirs = FileInputFormat.getInputPaths(job);
+ if (dirs.length == 0) {
+ throw new IOException("No input paths specified in job");
+ }
+ JobConf newjob = new JobConf(job);
+ ArrayList result = new ArrayList();
+
+ // for each dir, get the InputFormat, and do getSplits.
+ PartitionDesc part;
+ for (Path dir : dirs) {
+ part = HiveFileFormatUtils
+ .getPartitionDescFromPathRecursively(pathToPartitionInfo, dir,
+ IOPrepareCache.get().allocatePartitionDescMap(), true);
+ // create a new InputFormat instance if this is the first time to see this
+ // class
+ Class inputFormatClass = part.getInputFileFormatClass();
+ InputFormat inputFormat = getInputFormatFromCache(inputFormatClass, job);
+ Utilities.copyTableJobPropertiesToConf(part.getTableDesc(), newjob);
+
+ FileInputFormat.setInputPaths(newjob, dir);
+ newjob.setInputFormat(inputFormat.getClass());
+ InputSplit[] iss = inputFormat.getSplits(newjob, numSplits / dirs.length);
+ for (InputSplit is : iss) {
+ result.add(new HiveInputSplit(is, inputFormatClass.getName()));
+ }
+ }
+ return result.toArray(new HiveInputSplit[result.size()]);
+ }
+
+ @Override
+ public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
+ String indexFileStr = job.get(indexFile);
+ l4j.info("index_file is " + indexFileStr);
+
+ HiveIndexResult hiveIndexResult = null;
+ if (indexFileStr != null) {
+ try {
+ hiveIndexResult = new HiveIndexResult(indexFileStr, job);
+ } catch (HiveException e) {
+ l4j.error("Unable to read index..");
+ throw new IOException(e);
+ }
+
+ Set inputFiles = hiveIndexResult.buckets.keySet();
+ Iterator iter = inputFiles.iterator();
+ boolean first = true;
+ StringBuilder newInputPaths = new StringBuilder();
+ while(iter.hasNext()) {
+ String path = iter.next();
+ if (path.trim().equalsIgnoreCase("")) {
+ continue;
+ }
+ if (!first) {
+ newInputPaths.append(",");
+ } else {
+ first = false;
+ }
+ newInputPaths.append(path);
+ }
+
+ FileInputFormat.setInputPaths(job, newInputPaths.toString());
+ } else {
+ return super.getSplits(job, numSplits);
+ }
+
+ HiveInputSplit[] splits = (HiveInputSplit[]) this.doGetSplits(job, numSplits);
+
+ ArrayList newSplits = new ArrayList(
+ numSplits);
+ for (HiveInputSplit split : splits) {
+ l4j.info("split start : " + split.getStart());
+ l4j.info("split end : " + (split.getStart() + split.getLength()));
+
+ try {
+ if (hiveIndexResult.contains(split)) {
+ // we may miss a sync here
+ HiveInputSplit newSplit = split;
+ if (split.inputFormatClassName().contains("RCFile")
+ || split.inputFormatClassName().contains("SequenceFile")) {
+ if (split.getStart() > SequenceFile.SYNC_INTERVAL) {
+ newSplit = new HiveInputSplit(new FileSplit(split.getPath(), split
+ .getStart()
+ - SequenceFile.SYNC_INTERVAL, split.getLength()
+ + SequenceFile.SYNC_INTERVAL, split.getLocations()), split
+ .inputFormatClassName());
+ }
+ }
+ newSplits.add(newSplit);
+ }
+ } catch (HiveException e) {
+ throw new RuntimeException(
+ "Unable to get metadata for input table split" + split.getPath());
+ }
+ }
+ InputSplit retA[] = newSplits.toArray((new FileSplit[newSplits.size()]));
+ l4j.info("Number of input splits: " + splits.length + " new input splits: "
+ + retA.length);
+ return retA;
+ }
+}
diff --git ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java
new file mode 100644
index 0000000..b30a1f1
--- /dev/null
+++ ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.index;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.api.StageType;
+
+public class IndexMetadataChangeTask extends Task{
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ protected int execute(DriverContext driverContext) {
+
+ try {
+ Hive db = Hive.get(conf);
+ IndexMetadataChangeWork work = this.getWork();
+ String tblName = work.getIndexTbl();
+ Table tbl = db.getTable(work.getDbName(), tblName);
+ if (tbl == null ) {
+ console.printError("Index table can not be null.");
+ return 1;
+ }
+
+ if (!tbl.getTableType().equals(TableType.INDEX_TABLE)) {
+ console.printError("Table " + tbl.getTableName() + " not specified.");
+ return 1;
+ }
+
+ if (tbl.isPartitioned() && work.getPartSpec() == null) {
+ console.printError("Index table is partitioned, but no partition specified.");
+ return 1;
+ }
+
+ if (work.getPartSpec() != null) {
+ Partition part = db.getPartition(tbl, work.getPartSpec(), false);
+ if (part == null) {
+ console.printError("Partition " +
+ Warehouse.makePartName(work.getPartSpec(), false).toString()
+ + " does not exist.");
+ return 1;
+ }
+
+ Path url = new Path(part.getDataLocation().toString());
+ FileSystem fs = url.getFileSystem(conf);
+ FileStatus fstat = fs.getFileStatus(url);
+
+ part.getParameters().put(HiveIndex.INDEX_TABLE_CREATETIME, Long.toString(fstat.getModificationTime()));
+ db.alterPartition(tbl.getTableName(), part);
+ } else {
+ Path url = new Path(tbl.getDataLocation().toString());
+ FileSystem fs = url.getFileSystem(conf);
+ FileStatus fstat = fs.getFileStatus(url);
+ tbl.getParameters().put(HiveIndex.INDEX_TABLE_CREATETIME, Long.toString(fstat.getModificationTime()));
+ db.alterTable(tbl.getTableName(), tbl);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ console.printError("Error changing index table/partition metadata "
+ + e.getMessage());
+ return 1;
+ }
+ return 0;
+ }
+
+ @Override
+ public String getName() {
+ return IndexMetadataChangeTask.class.getSimpleName();
+ }
+
+ @Override
+ public StageType getType() {
+ return StageType.DDL;
+ }
+
+ @Override
+ protected void localizeMRTmpFilesImpl(Context ctx) {
+ }
+
+}
diff --git ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeWork.java ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeWork.java
new file mode 100644
index 0000000..59a9bd6
--- /dev/null
+++ ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeWork.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.index;
+
+import java.io.Serializable;
+import java.util.HashMap;
+
+public class IndexMetadataChangeWork implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private HashMap partSpec;
+ private String indexTbl;
+ private String dbName;
+
+ public IndexMetadataChangeWork() {
+ }
+
+ public IndexMetadataChangeWork(HashMap partSpec,
+ String indexTbl, String dbName) {
+ super();
+ this.partSpec = partSpec;
+ this.indexTbl = indexTbl;
+ this.dbName = dbName;
+ }
+
+ public HashMap getPartSpec() {
+ return partSpec;
+ }
+
+ public void setPartSpec(HashMap partSpec) {
+ this.partSpec = partSpec;
+ }
+
+ public String getIndexTbl() {
+ return indexTbl;
+ }
+
+ public void setIndexTbl(String indexTbl) {
+ this.indexTbl = indexTbl;
+ }
+
+ public String getDbName() {
+ return dbName;
+ }
+
+ public void setDbName(String dbName) {
+ this.dbName = dbName;
+ }
+
+}
diff --git ql/src/java/org/apache/hadoop/hive/ql/index/TableBasedIndexHandler.java ql/src/java/org/apache/hadoop/hive/ql/index/TableBasedIndexHandler.java
new file mode 100644
index 0000000..02ab78c
--- /dev/null
+++ ql/src/java/org/apache/hadoop/hive/ql/index/TableBasedIndexHandler.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.index;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Set;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.hooks.ReadEntity;
+import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.HiveUtils;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.PartitionDesc;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
+
+/**
+ * Index handler for indexes that use tables to store indexes.
+ */
+public abstract class TableBasedIndexHandler extends AbstractIndexHandler {
+ protected Configuration configuration;
+
+ @Override
+ public List> generateIndexBuildTaskList(
+ org.apache.hadoop.hive.ql.metadata.Table baseTbl,
+ org.apache.hadoop.hive.metastore.api.Index index,
+ List indexTblPartitions, List baseTblPartitions,
+ org.apache.hadoop.hive.ql.metadata.Table indexTbl,
+ Set inputs, Set outputs) throws HiveException {
+ try {
+
+ TableDesc desc = Utilities.getTableDesc(indexTbl);
+
+ List newBaseTblPartitions = new ArrayList();
+
+ List> indexBuilderTasks = new ArrayList>();
+
+ if (!baseTbl.isPartitioned()) {
+ // the table does not have any partition, then create index for the
+ // whole table
+ Task> indexBuilder = getIndexBuilderMapRedTask(inputs, outputs, index.getSd().getCols(), false,
+ new PartitionDesc(desc, null), indexTbl.getTableName(),
+ new PartitionDesc(Utilities.getTableDesc(baseTbl), null),
+ baseTbl.getTableName(), indexTbl.getDbName());
+ indexBuilderTasks.add(indexBuilder);
+ } else {
+
+ // check whether the index table partitions are still exists in base
+ // table
+ for (int i = 0; i < indexTblPartitions.size(); i++) {
+ Partition indexPart = indexTblPartitions.get(i);
+ Partition basePart = null;
+ for (int j = 0; j < baseTblPartitions.size(); j++) {
+ if (baseTblPartitions.get(j).getName().equals(indexPart.getName())) {
+ basePart = baseTblPartitions.get(j);
+ newBaseTblPartitions.add(baseTblPartitions.get(j));
+ break;
+ }
+ }
+ if (basePart == null) {
+ throw new RuntimeException(
+ "Partitions of base table and index table are inconsistent.");
+ }
+ // for each partition, spawn a map reduce task.
+ Task> indexBuilder = getIndexBuilderMapRedTask(inputs, outputs, index.getSd().getCols(), true,
+ new PartitionDesc(indexPart), indexTbl.getTableName(),
+ new PartitionDesc(basePart), baseTbl.getTableName(), indexTbl.getDbName());
+ indexBuilderTasks.add(indexBuilder);
+ }
+ }
+ return indexBuilderTasks;
+ } catch (Exception e) {
+ throw new SemanticException(e);
+ }
+ }
+
+ abstract protected Task> getIndexBuilderMapRedTask(Set inputs, Set outputs,
+ List indexField, boolean partitioned,
+ PartitionDesc indexTblPartDesc, String indexTableName,
+ PartitionDesc baseTablePartDesc, String baseTableName, String dbName);
+
+ protected List getPartKVPairStringArray(
+ LinkedHashMap partSpec) {
+ List ret = new ArrayList(partSpec.size());
+ Iterator> iter = partSpec.entrySet().iterator();
+ while (iter.hasNext()) {
+ StringBuilder sb = new StringBuilder();
+ Entry p = iter.next();
+ sb.append(HiveUtils.unparseIdentifier(p.getKey()));
+ sb.append(" = ");
+ sb.append("'");
+ sb.append(HiveUtils.escapeString(p.getValue()));
+ sb.append("'");
+ ret.add(sb.toString());
+ }
+ return ret;
+ }
+
+ @Override
+ public boolean usesIndexTable() {
+ return true;
+ }
+
+ @Override
+ public Configuration getConf() {
+ return configuration;
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ this.configuration = conf;
+ }
+
+}
diff --git ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java
new file mode 100644
index 0000000..af9d7b1
--- /dev/null
+++ ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.index.bitmap;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Set;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Index;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.hooks.ReadEntity;
+import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.index.TableBasedIndexHandler;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.HiveUtils;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.PartitionDesc;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
+import org.apache.hadoop.hive.ql.index.IndexMetadataChangeTask;
+import org.apache.hadoop.hive.ql.index.IndexMetadataChangeWork;
+
+/**
+ * Index handler for the bitmap index. Bitmap index uses an EWAH-compressed
+ * bitmap to represent the values in a table.
+ */
+public class BitmapIndexHandler extends TableBasedIndexHandler {
+
+ @Override
+ public void analyzeIndexDefinition(Table baseTable, Index index,
+ Table indexTable) throws HiveException {
+ StorageDescriptor storageDesc = index.getSd();
+ if (this.usesIndexTable() && indexTable != null) {
+ StorageDescriptor indexTableSd = storageDesc.deepCopy();
+ List indexTblCols = indexTableSd.getCols();
+ FieldSchema bucketFileName = new FieldSchema("_bucketname", "string", "");
+ indexTblCols.add(bucketFileName);
+ FieldSchema offSets = new FieldSchema("_offset", "bigint", "");
+ indexTblCols.add(offSets);
+ FieldSchema bitmaps = new FieldSchema("_bitmaps", "array", "");
+ indexTblCols.add(bitmaps);
+ indexTable.setSd(indexTableSd);
+ }
+ }
+
+ @Override
+ protected Task> getIndexBuilderMapRedTask(Set inputs, Set outputs,
+ List indexField, boolean partitioned,
+ PartitionDesc indexTblPartDesc, String indexTableName,
+ PartitionDesc baseTablePartDesc, String baseTableName, String dbName) {
+
+ HiveConf conf = new HiveConf(getConf(), BitmapIndexHandler.class);
+ HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVEROWOFFSET, true);
+
+ String indexCols = HiveUtils.getUnparsedColumnNamesFromFieldSchema(indexField);
+
+ //form a new insert overwrite query.
+ StringBuilder command= new StringBuilder();
+ LinkedHashMap partSpec = indexTblPartDesc.getPartSpec();
+
+ command.append("INSERT OVERWRITE TABLE " + HiveUtils.unparseIdentifier(indexTableName ));
+ if (partitioned && indexTblPartDesc != null) {
+ command.append(" PARTITION ( ");
+ List ret = getPartKVPairStringArray(partSpec);
+ for (int i = 0; i < ret.size(); i++) {
+ String partKV = ret.get(i);
+ command.append(partKV);
+ if (i < ret.size() - 1) {
+ command.append(",");
+ }
+ }
+ command.append(" ) ");
+ }
+
+ command.append(" SELECT ");
+ command.append(indexCols);
+ command.append(",");
+ command.append(VirtualColumn.FILENAME.getName());
+ command.append(",");
+ command.append(VirtualColumn.BLOCKOFFSET.getName());
+ command.append(",");
+ command.append("EWAH_BITMAP(");
+ command.append(VirtualColumn.ROWOFFSET.getName());
+ command.append(")");
+ command.append(" FROM " + HiveUtils.unparseIdentifier(baseTableName) );
+ LinkedHashMap basePartSpec = baseTablePartDesc.getPartSpec();
+ if(basePartSpec != null) {
+ command.append(" WHERE ");
+ List pkv = getPartKVPairStringArray(basePartSpec);
+ for (int i = 0; i < pkv.size(); i++) {
+ String partKV = pkv.get(i);
+ command.append(partKV);
+ if (i < pkv.size() - 1) {
+ command.append(" AND ");
+ }
+ }
+ }
+ command.append(" GROUP BY ");
+ command.append(VirtualColumn.FILENAME.getName());
+ command.append(",");
+ command.append(VirtualColumn.BLOCKOFFSET.getName());
+ for (FieldSchema fieldSchema : indexField) {
+ command.append(",");
+ command.append(HiveUtils.unparseIdentifier(fieldSchema.getName()));
+ }
+
+ // Require clusterby ROWOFFSET if map-size aggregation is off.
+ if (!configuration.get("hive.map.aggr", null).equals("true")) {
+ command.append(" CLUSTER BY ");
+ command.append(VirtualColumn.ROWOFFSET.getName());
+ }
+
+ Driver driver = new Driver(conf);
+ driver.compile(command.toString());
+
+ Task> rootTask = driver.getPlan().getRootTasks().get(0);
+ inputs.addAll(driver.getPlan().getInputs());
+ outputs.addAll(driver.getPlan().getOutputs());
+
+ IndexMetadataChangeWork indexMetaChange = new IndexMetadataChangeWork(partSpec, indexTableName, dbName);
+ IndexMetadataChangeTask indexMetaChangeTsk = new IndexMetadataChangeTask();
+ indexMetaChangeTsk.setWork(indexMetaChange);
+ rootTask.addDependentTask(indexMetaChangeTsk);
+
+ return rootTask;
+ }
+}
diff --git ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapObjectInput.java ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapObjectInput.java
new file mode 100644
index 0000000..75f66a4
--- /dev/null
+++ ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapObjectInput.java
@@ -0,0 +1,182 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.index.bitmap;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.lazy.LazyLong;
+
+/**
+ * An ObjectInput that allows for conversion from an List of LongWritable
+ * to an EWAH-compressed bitmap.
+ */
+public class BitmapObjectInput implements ObjectInput {
+ Iterator bufferIter;
+ List buffer;
+
+ public BitmapObjectInput() {
+ buffer = new ArrayList();
+ bufferIter = buffer.iterator();
+ }
+
+ public BitmapObjectInput(List l) {
+ readFromList(l);
+ }
+
+ public void readFromList(List l) {
+ buffer = l;
+ bufferIter = buffer.iterator();
+ }
+
+ @Override
+ public int available() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void close() throws IOException {
+ throw new UnsupportedOperationException();
+
+ }
+
+ @Override
+ public int read() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int read(byte[] arg0) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int read(byte[] arg0, int arg1, int arg2) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object readObject() throws ClassNotFoundException, IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long skip(long arg0) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean readBoolean() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public byte readByte() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public char readChar() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public double readDouble() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public float readFloat() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void readFully(byte[] arg0) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void readFully(byte[] arg0, int arg1, int arg2) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int readInt() throws IOException {
+ if (bufferIter.hasNext()) {
+ LongObjectInspector loi = PrimitiveObjectInspectorFactory.writableLongObjectInspector;
+ Long l = PrimitiveObjectInspectorUtils.getLong(bufferIter.next(), loi);
+ return l.intValue();
+ //return bufferIter.next().intValue();
+ }
+ else {
+ throw new IOException();
+ }
+ }
+
+ @Override
+ public String readLine() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long readLong() throws IOException {
+ //LongObjectInspector loi = PrimitiveObjectInspectorFactory.writableLongObjectInspector;
+ if (bufferIter.hasNext()) {
+ LongObjectInspector loi = PrimitiveObjectInspectorFactory.writableLongObjectInspector;
+ return PrimitiveObjectInspectorUtils.getLong(bufferIter.next(), loi);
+ //return bufferIter.next();
+ }
+ else {
+ throw new IOException();
+ }
+ }
+
+ @Override
+ public short readShort() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String readUTF() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int readUnsignedByte() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int readUnsignedShort() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int skipBytes(int n) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+
+}
diff --git ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapObjectOutput.java ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapObjectOutput.java
new file mode 100644
index 0000000..7a4b9d8
--- /dev/null
+++ ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapObjectOutput.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.index.bitmap;
+
+import java.io.IOException;
+import java.io.ObjectOutput;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+
+/**
+ * An ObjectOutput that allows conversion from an EWAH-compressed bitmap
+ * to an List of LongWritable.
+ */
+public class BitmapObjectOutput implements ObjectOutput {
+ ArrayList buffer = new ArrayList();
+
+ public List list() {
+ return buffer;
+ }
+
+ @Override
+ public void close() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void flush() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void write(int arg0) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void write(byte[] arg0) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void write(byte[] arg0, int arg1, int arg2) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void writeObject(Object arg0) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void writeBoolean(boolean arg0) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void writeByte(int arg0) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void writeBytes(String arg0) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void writeChar(int arg0) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void writeChars(String arg0) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void writeDouble(double v) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void writeFloat(float v) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void writeInt(int v) throws IOException {
+ buffer.add(new LongWritable(v));
+ }
+
+ @Override
+ public void writeLong(long v) throws IOException {
+ buffer.add(new LongWritable(v));
+ }
+
+ @Override
+ public void writeShort(int v) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void writeUTF(String s) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+}
diff --git ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java
index 1f01446..f90d64f 100644
--- ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java
+++ ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.hive.ql.index.compact;
import java.util.ArrayList;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
@@ -28,7 +27,6 @@ import java.util.Map.Entry;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Index;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
@@ -38,8 +36,9 @@ import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
-import org.apache.hadoop.hive.ql.index.AbstractIndexHandler;
-import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.index.TableBasedIndexHandler;
+import org.apache.hadoop.hive.ql.index.IndexMetadataChangeTask;
+import org.apache.hadoop.hive.ql.index.IndexMetadataChangeWork;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.HiveUtils;
import org.apache.hadoop.hive.ql.metadata.Partition;
@@ -48,8 +47,8 @@ import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.PartitionDesc;
import org.apache.hadoop.hive.ql.plan.TableDesc;
-public class CompactIndexHandler extends AbstractIndexHandler {
-
+public class CompactIndexHandler extends TableBasedIndexHandler {
+
private Configuration configuration;
@Override
@@ -68,63 +67,11 @@ public class CompactIndexHandler extends AbstractIndexHandler {
}
@Override
- public List> generateIndexBuildTaskList(
- org.apache.hadoop.hive.ql.metadata.Table baseTbl,
- org.apache.hadoop.hive.metastore.api.Index index,
- List indexTblPartitions, List baseTblPartitions,
- org.apache.hadoop.hive.ql.metadata.Table indexTbl,
- Set inputs, Set outputs) throws HiveException {
- try {
-
- TableDesc desc = Utilities.getTableDesc(indexTbl);
-
- List newBaseTblPartitions = new ArrayList();
-
- List> indexBuilderTasks = new ArrayList>();
-
- if (!baseTbl.isPartitioned()) {
- // the table does not have any partition, then create index for the
- // whole table
- Task> indexBuilder = getIndexBuilderMapRedTask(inputs, outputs, index.getSd().getCols(), false,
- new PartitionDesc(desc, null), indexTbl.getTableName(),
- new PartitionDesc(Utilities.getTableDesc(baseTbl), null),
- baseTbl.getTableName(), indexTbl.getDbName());
- indexBuilderTasks.add(indexBuilder);
- } else {
-
- // check whether the index table partitions are still exists in base
- // table
- for (int i = 0; i < indexTblPartitions.size(); i++) {
- Partition indexPart = indexTblPartitions.get(i);
- Partition basePart = null;
- for (int j = 0; j < baseTblPartitions.size(); j++) {
- if (baseTblPartitions.get(j).getName().equals(indexPart.getName())) {
- basePart = baseTblPartitions.get(j);
- newBaseTblPartitions.add(baseTblPartitions.get(j));
- break;
- }
- }
- if (basePart == null)
- throw new RuntimeException(
- "Partitions of base table and index table are inconsistent.");
- // for each partition, spawn a map reduce task.
- Task> indexBuilder = getIndexBuilderMapRedTask(inputs, outputs, index.getSd().getCols(), true,
- new PartitionDesc(indexPart), indexTbl.getTableName(),
- new PartitionDesc(basePart), baseTbl.getTableName(), indexTbl.getDbName());
- indexBuilderTasks.add(indexBuilder);
- }
- }
- return indexBuilderTasks;
- } catch (Exception e) {
- throw new SemanticException(e);
- }
- }
-
- private Task> getIndexBuilderMapRedTask(Set inputs, Set outputs,
+ protected Task> getIndexBuilderMapRedTask(Set inputs, Set outputs,
List indexField, boolean partitioned,
PartitionDesc indexTblPartDesc, String indexTableName,
PartitionDesc baseTablePartDesc, String baseTableName, String dbName) {
-
+
String indexCols = HiveUtils.getUnparsedColumnNamesFromFieldSchema(indexField);
//form a new insert overwrite query.
@@ -138,12 +85,13 @@ public class CompactIndexHandler extends AbstractIndexHandler {
for (int i = 0; i < ret.size(); i++) {
String partKV = ret.get(i);
command.append(partKV);
- if (i < ret.size() - 1)
+ if (i < ret.size() - 1) {
command.append(",");
+ }
}
command.append(" ) ");
}
-
+
command.append(" SELECT ");
command.append(indexCols);
command.append(",");
@@ -161,8 +109,9 @@ public class CompactIndexHandler extends AbstractIndexHandler {
for (int i = 0; i < pkv.size(); i++) {
String partKV = pkv.get(i);
command.append(partKV);
- if (i < pkv.size() - 1)
+ if (i < pkv.size() - 1) {
command.append(" AND ");
+ }
}
}
command.append(" GROUP BY ");
@@ -174,45 +123,12 @@ public class CompactIndexHandler extends AbstractIndexHandler {
Task> rootTask = driver.getPlan().getRootTasks().get(0);
inputs.addAll(driver.getPlan().getInputs());
outputs.addAll(driver.getPlan().getOutputs());
-
+
IndexMetadataChangeWork indexMetaChange = new IndexMetadataChangeWork(partSpec, indexTableName, dbName);
- IndexMetadataChangeTask indexMetaChangeTsk = new IndexMetadataChangeTask();
+ IndexMetadataChangeTask indexMetaChangeTsk = new IndexMetadataChangeTask();
indexMetaChangeTsk.setWork(indexMetaChange);
rootTask.addDependentTask(indexMetaChangeTsk);
return rootTask;
}
-
- private List getPartKVPairStringArray(
- LinkedHashMap partSpec) {
- List ret = new ArrayList(partSpec.size());
- Iterator> iter = partSpec.entrySet().iterator();
- while (iter.hasNext()) {
- StringBuilder sb = new StringBuilder();
- Entry p = iter.next();
- sb.append(HiveUtils.unparseIdentifier(p.getKey()));
- sb.append(" = ");
- sb.append("'");
- sb.append(p.getValue());
- sb.append("'");
- ret.add(sb.toString());
- }
- return ret;
- }
-
- @Override
- public boolean usesIndexTable() {
- return true;
- }
-
- @Override
- public Configuration getConf() {
- return configuration;
- }
-
- @Override
- public void setConf(Configuration conf) {
- this.configuration = conf;
- }
-
}
diff --git ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexInputFormat.java ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexInputFormat.java
index 6c320c5..7cebe68 100644
--- ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexInputFormat.java
+++ ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexInputFormat.java
@@ -15,136 +15,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.hive.ql.index.compact;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.Set;
+package org.apache.hadoop.hive.ql.index.compact;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
-import org.apache.hadoop.hive.ql.io.HiveInputFormat;
-import org.apache.hadoop.hive.ql.io.IOPrepareCache;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.plan.PartitionDesc;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.FileSplit;
-import org.apache.hadoop.mapred.InputFormat;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.hive.ql.index.HiveIndexedInputFormat;
-public class HiveCompactIndexInputFormat extends HiveInputFormat {
+public class HiveCompactIndexInputFormat extends HiveIndexedInputFormat {
- public static final Log l4j = LogFactory.getLog("HiveIndexInputFormat");
+ public static final Log l4j =
+ LogFactory.getLog(HiveCompactIndexInputFormat.class.getSimpleName());
public HiveCompactIndexInputFormat() {
- super();
- }
-
- public InputSplit[] doGetSplits(JobConf job, int numSplits) throws IOException {
-
- super.init(job);
-
- Path[] dirs = FileInputFormat.getInputPaths(job);
- if (dirs.length == 0) {
- throw new IOException("No input paths specified in job");
- }
- JobConf newjob = new JobConf(job);
- ArrayList result = new ArrayList();
-
- // for each dir, get the InputFormat, and do getSplits.
- for (Path dir : dirs) {
- PartitionDesc part = HiveFileFormatUtils
- .getPartitionDescFromPathRecursively(pathToPartitionInfo, dir,
- IOPrepareCache.get().allocatePartitionDescMap(), true);
- // create a new InputFormat instance if this is the first time to see this
- // class
- Class inputFormatClass = part.getInputFileFormatClass();
- InputFormat inputFormat = getInputFormatFromCache(inputFormatClass, job);
- Utilities.copyTableJobPropertiesToConf(part.getTableDesc(), newjob);
-
- FileInputFormat.setInputPaths(newjob, dir);
- newjob.setInputFormat(inputFormat.getClass());
- InputSplit[] iss = inputFormat.getSplits(newjob, numSplits / dirs.length);
- for (InputSplit is : iss) {
- result.add(new HiveInputSplit(is, inputFormatClass.getName()));
- }
- }
- return result.toArray(new HiveInputSplit[result.size()]);
- }
-
- @Override
- public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
- String indexFileStr = job.get("hive.index.compact.file");
- l4j.info("index_file is " + indexFileStr);
-
- HiveCompactIndexResult hiveIndexResult = null;
- if (indexFileStr != null) {
- try {
- hiveIndexResult = new HiveCompactIndexResult(indexFileStr, job);
- } catch (HiveException e) {
- l4j.error("Unable to read index..");
- throw new IOException(e);
- }
-
- Set inputFiles = hiveIndexResult.buckets.keySet();
- Iterator iter = inputFiles.iterator();
- boolean first = true;
- StringBuilder newInputPaths = new StringBuilder();
- while(iter.hasNext()) {
- String path = iter.next();
- if (path.trim().equalsIgnoreCase(""))
- continue;
- if (!first) {
- newInputPaths.append(",");
- } else {
- first = false;
- }
- newInputPaths.append(path);
- }
-
- FileInputFormat.setInputPaths(job, newInputPaths.toString());
- } else {
- return super.getSplits(job, numSplits);
- }
-
- HiveInputSplit[] splits = (HiveInputSplit[]) this.doGetSplits(job, numSplits);
-
- ArrayList newSplits = new ArrayList(
- numSplits);
- for (HiveInputSplit split : splits) {
- l4j.info("split start : " + split.getStart());
- l4j.info("split end : " + (split.getStart() + split.getLength()));
-
- try {
- if (hiveIndexResult.contains(split)) {
- // we may miss a sync here
- HiveInputSplit newSplit = split;
- if (split.inputFormatClassName().contains("RCFile")
- || split.inputFormatClassName().contains("SequenceFile")) {
- if (split.getStart() > SequenceFile.SYNC_INTERVAL) {
- newSplit = new HiveInputSplit(new FileSplit(split.getPath(), split
- .getStart()
- - SequenceFile.SYNC_INTERVAL, split.getLength()
- + SequenceFile.SYNC_INTERVAL, split.getLocations()), split
- .inputFormatClassName());
- }
- }
- newSplits.add(newSplit);
- }
- } catch (HiveException e) {
- throw new RuntimeException(
- "Unable to get metadata for input table split" + split.getPath());
- }
- }
- InputSplit retA[] = newSplits.toArray((new FileSplit[newSplits.size()]));
- l4j.info("Number of input splits: " + splits.length + " new input splits: "
- + retA.length);
- return retA;
+ super("hive.index.compact.file");
}
}
diff --git ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexResult.java ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexResult.java
deleted file mode 100644
index 0c9ccea..0000000
--- ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexResult.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.index.compact;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.serde2.columnar.BytesRefWritable;
-import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.FileSplit;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.LineRecordReader.LineReader;
-
-public class HiveCompactIndexResult {
-
- public static final Log l4j = LogFactory.getLog("HiveCompactIndexResult");
-
- // IndexBucket
- static class IBucket {
- private String name = null;
- private SortedSet offsets = new TreeSet();
-
- public IBucket(String n) {
- name = n;
- }
-
- public void add(Long offset) {
- offsets.add(offset);
- }
-
- public String getName() {
- return name;
- }
-
- public SortedSet getOffsets() {
- return offsets;
- }
-
- public boolean equals(Object obj) {
- if (obj.getClass() != this.getClass()) {
- return false;
- }
- return (((IBucket) obj).name.compareToIgnoreCase(this.name) == 0);
- }
- }
-
- JobConf job = null;
- BytesRefWritable[] bytesRef = new BytesRefWritable[2];
- boolean ignoreHdfsLoc = false;
-
- public HiveCompactIndexResult(String indexFile, JobConf conf) throws IOException,
- HiveException {
- job = conf;
-
- bytesRef[0] = new BytesRefWritable();
- bytesRef[1] = new BytesRefWritable();
- ignoreHdfsLoc = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_INDEX_IGNORE_HDFS_LOC);
-
- if (indexFile != null) {
- Path indexFilePath = new Path(indexFile);
- FileSystem fs = FileSystem.get(conf);
- FileStatus indexStat = fs.getFileStatus(indexFilePath);
- List paths = new ArrayList();
- if (indexStat.isDir()) {
- FileStatus[] fss = fs.listStatus(indexFilePath);
- for (FileStatus f : fss) {
- paths.add(f.getPath());
- }
- } else {
- paths.add(indexFilePath);
- }
-
- for (Path indexFinalPath : paths) {
- FSDataInputStream ifile = fs.open(indexFinalPath);
- LineReader lr = new LineReader(ifile, conf);
- Text line = new Text();
- while (lr.readLine(line) > 0) {
- add(line);
- }
- // this will close the input stream
- lr.close();
- }
- }
- }
-
- Map buckets = new HashMap();
-
- private void add(Text line) throws HiveException {
- String l = line.toString();
- byte[] bytes = l.getBytes();
- int firstEnd = 0;
- int i = 0;
- for (int index = 0; index < bytes.length; index++) {
- if (bytes[index] == LazySimpleSerDe.DefaultSeparators[0]) {
- i++;
- firstEnd = index;
- }
- }
- if (i > 1) {
- throw new HiveException(
- "Bad index file row (index file should only contain two columns: bucket_file_name and offset lists.) ."
- + line.toString());
- }
- String bucketFileName = new String(bytes, 0, firstEnd);
-
- if (ignoreHdfsLoc) {
- Path tmpPath = new Path(bucketFileName);
- bucketFileName = tmpPath.toUri().getPath();
- }
- IBucket bucket = buckets.get(bucketFileName);
- if (bucket == null) {
- bucket = new IBucket(bucketFileName);
- buckets.put(bucketFileName, bucket);
- }
-
- int currentStart = firstEnd + 1;
- int currentEnd = firstEnd + 1;
- for (; currentEnd < bytes.length; currentEnd++) {
- if (bytes[currentEnd] == LazySimpleSerDe.DefaultSeparators[1]) {
- String one_offset = new String(bytes, currentStart, currentEnd
- - currentStart);
- Long offset = Long.parseLong(one_offset);
- bucket.getOffsets().add(offset);
- currentStart = currentEnd + 1;
- }
- }
- String one_offset = new String(bytes, currentStart, currentEnd
- - currentStart);
- bucket.getOffsets().add(Long.parseLong(one_offset));
- }
-
- public boolean contains(FileSplit split) throws HiveException {
-
- if (buckets == null) {
- return false;
- }
- String bucketName = split.getPath().toString();
- IBucket bucket = buckets.get(bucketName);
- if (bucket == null) {
- bucketName = split.getPath().toUri().getPath();
- bucket = buckets.get(bucketName);
- if (bucket == null) {
- return false;
- }
- }
-
- for (Long offset : bucket.getOffsets()) {
- if ((offset >= split.getStart())
- && (offset <= split.getStart() + split.getLength())) {
- return true;
- }
- }
- return false;
- }
-}
diff --git ql/src/java/org/apache/hadoop/hive/ql/index/compact/IndexMetadataChangeTask.java ql/src/java/org/apache/hadoop/hive/ql/index/compact/IndexMetadataChangeTask.java
deleted file mode 100644
index eac168f..0000000
--- ql/src/java/org/apache/hadoop/hive/ql/index/compact/IndexMetadataChangeTask.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.index.compact;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.metastore.TableType;
-import org.apache.hadoop.hive.metastore.Warehouse;
-import org.apache.hadoop.hive.ql.Context;
-import org.apache.hadoop.hive.ql.DriverContext;
-import org.apache.hadoop.hive.ql.exec.Task;
-import org.apache.hadoop.hive.ql.index.HiveIndex;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.Partition;
-import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.plan.api.StageType;
-
-public class IndexMetadataChangeTask extends Task{
-
- private static final long serialVersionUID = 1L;
-
- @Override
- protected int execute(DriverContext driverContext) {
-
- try {
- Hive db = Hive.get(conf);
- IndexMetadataChangeWork work = this.getWork();
- String tblName = work.getIndexTbl();
- Table tbl = db.getTable(work.getDbName(), tblName);
- if (tbl == null ) {
- console.printError("Index table can not be null.");
- return 1;
- }
-
- if (!tbl.getTableType().equals(TableType.INDEX_TABLE)) {
- console.printError("Table " + tbl.getTableName() + " not specified.");
- return 1;
- }
-
- if (tbl.isPartitioned() && work.getPartSpec() == null) {
- console.printError("Index table is partitioned, but no partition specified.");
- return 1;
- }
-
- if (work.getPartSpec() != null) {
- Partition part = db.getPartition(tbl, work.getPartSpec(), false);
- if (part == null) {
- console.printError("Partition " +
- Warehouse.makePartName(work.getPartSpec(), false).toString()
- + " does not exist.");
- return 1;
- }
-
- Path url = new Path(part.getDataLocation().toString());
- FileSystem fs = url.getFileSystem(conf);
- FileStatus fstat = fs.getFileStatus(url);
-
- part.getParameters().put(HiveIndex.INDEX_TABLE_CREATETIME, Long.toString(fstat.getModificationTime()));
- db.alterPartition(tbl.getTableName(), part);
- } else {
- Path url = new Path(tbl.getDataLocation().toString());
- FileSystem fs = url.getFileSystem(conf);
- FileStatus fstat = fs.getFileStatus(url);
- tbl.getParameters().put(HiveIndex.INDEX_TABLE_CREATETIME, Long.toString(fstat.getModificationTime()));
- db.alterTable(tbl.getTableName(), tbl);
- }
- } catch (Exception e) {
- e.printStackTrace();
- console.printError("Error changing index table/partition metadata "
- + e.getMessage());
- return 1;
- }
- return 0;
- }
-
- @Override
- public String getName() {
- return "IndexMetadataChangeTask";
- }
-
- @Override
- public StageType getType() {
- return StageType.DDL;
- }
-
- @Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- }
-
-}
diff --git ql/src/java/org/apache/hadoop/hive/ql/index/compact/IndexMetadataChangeWork.java ql/src/java/org/apache/hadoop/hive/ql/index/compact/IndexMetadataChangeWork.java
deleted file mode 100644
index 26beb4e..0000000
--- ql/src/java/org/apache/hadoop/hive/ql/index/compact/IndexMetadataChangeWork.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.index.compact;
-
-import java.io.Serializable;
-import java.util.HashMap;
-
-public class IndexMetadataChangeWork implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- private HashMap partSpec;
- private String indexTbl;
- private String dbName;
-
- public IndexMetadataChangeWork() {
- }
-
- public IndexMetadataChangeWork(HashMap partSpec,
- String indexTbl, String dbName) {
- super();
- this.partSpec = partSpec;
- this.indexTbl = indexTbl;
- this.dbName = dbName;
- }
-
- public HashMap getPartSpec() {
- return partSpec;
- }
-
- public void setPartSpec(HashMap partSpec) {
- this.partSpec = partSpec;
- }
-
- public String getIndexTbl() {
- return indexTbl;
- }
-
- public void setIndexTbl(String indexTbl) {
- this.indexTbl = indexTbl;
- }
-
- public String getDbName() {
- return dbName;
- }
-
- public void setDbName(String dbName) {
- this.dbName = dbName;
- }
-
-}
diff --git ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java
index 391e5de..6abc6cd 100644
--- ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java
+++ ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java
@@ -76,16 +76,26 @@ public abstract class HiveContextAwareRecordReader implements RecordReader
long pointerPos = this.getPos();
if (!ioCxtRef.isBlockPointer) {
ioCxtRef.currentBlockStart = pointerPos;
+ ioCxtRef.currentRow = 0;
return;
}
+ ioCxtRef.currentRow++;
+
if (ioCxtRef.nextBlockStart == -1) {
ioCxtRef.nextBlockStart = pointerPos;
+ ioCxtRef.currentRow = 0;
}
if (pointerPos != ioCxtRef.nextBlockStart) {
// the reader pointer has moved to the end of next block, or the end of
// current record.
+ ioCxtRef.currentRow = 0;
+
+ if (ioCxtRef.currentBlockStart == ioCxtRef.nextBlockStart) {
+ ioCxtRef.currentRow = 1;
+ }
+
ioCxtRef.currentBlockStart = ioCxtRef.nextBlockStart;
ioCxtRef.nextBlockStart = pointerPos;
}
@@ -133,4 +143,4 @@ public abstract class HiveContextAwareRecordReader implements RecordReader
}
this.initIOContext(blockStart, blockPointer, split.getPath().toString());
}
-}
\ No newline at end of file
+}
diff --git ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java
index 77220a1..8091970 100644
--- ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java
+++ ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java
@@ -41,6 +41,7 @@ public class IOContext {
long currentBlockStart;
long nextBlockStart;
+ long currentRow;
boolean isBlockPointer;
boolean ioExceptions;
@@ -49,6 +50,7 @@ public class IOContext {
public IOContext() {
this.currentBlockStart = 0;
this.nextBlockStart = -1;
+ this.currentRow = 0;
this.isBlockPointer = true;
this.ioExceptions = false;
}
@@ -69,6 +71,14 @@ public class IOContext {
this.nextBlockStart = nextBlockStart;
}
+ public long getCurrentRow() {
+ return currentRow;
+ }
+
+ public void setCurrentRow(long currentRow) {
+ this.currentRow = currentRow;
+ }
+
public boolean isBlockPointer() {
return isBlockPointer;
}
diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java
index 30714b8..de9a07e 100644
--- ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java
+++ ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java
@@ -19,8 +19,14 @@
package org.apache.hadoop.hive.ql.metadata;
import java.io.Serializable;
+import java.util.Map;
import java.util.HashMap;
+import java.util.List;
+import java.util.ArrayList;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
@@ -28,15 +34,10 @@ public class VirtualColumn implements Serializable {
private static final long serialVersionUID = 1L;
- public static HashMap registry = new HashMap();
-
public static VirtualColumn FILENAME = new VirtualColumn("INPUT__FILE__NAME", (PrimitiveTypeInfo)TypeInfoFactory.stringTypeInfo);
public static VirtualColumn BLOCKOFFSET = new VirtualColumn("BLOCK__OFFSET__INSIDE__FILE", (PrimitiveTypeInfo)TypeInfoFactory.longTypeInfo);
+ public static VirtualColumn ROWOFFSET = new VirtualColumn("ROW__OFFSET__INSIDE__BLOCK", (PrimitiveTypeInfo)TypeInfoFactory.longTypeInfo);
- static {
- registry.put(FILENAME.name, FILENAME);
- registry.put(BLOCKOFFSET.name, BLOCKOFFSET);
- }
private String name;
private PrimitiveTypeInfo typeInfo;
@@ -44,7 +45,7 @@ public class VirtualColumn implements Serializable {
public VirtualColumn() {
}
-
+
public VirtualColumn(String name, PrimitiveTypeInfo typeInfo) {
this(name, typeInfo, true);
}
@@ -54,6 +55,17 @@ public class VirtualColumn implements Serializable {
this.typeInfo = typeInfo;
this.isHidden = isHidden;
}
+
+ public static List getRegistry(Configuration conf) {
+ ArrayList l = new ArrayList();
+ l.add(BLOCKOFFSET);
+ l.add(FILENAME);
+ if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEROWOFFSET)) {
+ l.add(ROWOFFSET);
+ }
+
+ return l;
+ }
public PrimitiveTypeInfo getTypeInfo() {
return typeInfo;
@@ -95,4 +107,4 @@ public class VirtualColumn implements Serializable {
&& this.typeInfo.getTypeName().equals(c.getTypeInfo().getTypeName());
}
-}
\ No newline at end of file
+}
diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java
index 1d56c0a..202b510 100644
--- ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java
@@ -147,7 +147,7 @@ public class OpProcFactory {
fieldSchemaMap.put(col.getName(), col);
}
- Iterator vcs = VirtualColumn.registry.values().iterator();
+ Iterator vcs = VirtualColumn.getRegistry(pctx.getConf()).iterator();
while (vcs.hasNext()) {
VirtualColumn vc = vcs.next();
fieldSchemaMap.put(vc.getName(), new FieldSchema(vc.getName(),
diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index c41bb32..15fcc24 100644
--- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -5798,7 +5798,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
//put all virutal columns in RowResolver.
- Iterator vcs = VirtualColumn.registry.values().iterator();
+ Iterator vcs = VirtualColumn.getRegistry(conf).iterator();
//use a list for easy cumtomize
List vcList = new ArrayList();
while (vcs.hasNext()) {
diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFEWAHBitmapBop.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFEWAHBitmapBop.java
new file mode 100644
index 0000000..3151c8a
--- /dev/null
+++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFEWAHBitmapBop.java
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import javaewah.EWAHCompressedBitmap;
+
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
+import org.apache.hadoop.hive.ql.index.bitmap.BitmapObjectInput;
+import org.apache.hadoop.hive.ql.index.bitmap.BitmapObjectOutput;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableVoidObjectInspector;
+import org.apache.hadoop.io.LongWritable;
+
+/**
+ * An abstract class for a UDF that performs a binary operation between two EWAH-compressed bitmaps.
+ * For example: Bitmap OR and AND operations between two EWAH-compressed bitmaps.
+ */
+abstract public class AbstractGenericUDFEWAHBitmapBop extends GenericUDF {
+ protected final ArrayList
@@ -196,11 +196,11 @@
location
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
transient_lastDdlTime
- 1300721798
+ 1303627883
@@ -258,11 +258,11 @@
location
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
transient_lastDdlTime
- 1300721798
+ 1303627883
@@ -521,7 +521,8 @@
sum
-
+
+ org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator$Mode
PARTIAL1
@@ -595,7 +596,8 @@
0.9
-
+
+ org.apache.hadoop.hive.ql.plan.GroupByDesc$Mode
HASH
@@ -891,7 +893,7 @@
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
src
@@ -903,7 +905,7 @@
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
src
@@ -960,11 +962,11 @@
location
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
transient_lastDdlTime
- 1300721798
+ 1303627883
@@ -1022,11 +1024,11 @@
location
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
transient_lastDdlTime
- 1300721798
+ 1303627883
@@ -1055,7 +1057,7 @@
1
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/scratchdir/hive_2011-03-21_08-36-38_795_2045194704724889129/-ext-10000
+ pfile:/home/mwang/Projects/hive-git/build/ql/scratchdir/hive_2011-04-23_23-51-25_951_1689143600430803950/-ext-10000
true
@@ -1064,7 +1066,7 @@
1
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/scratchdir/hive_2011-03-21_08-36-38_795_2045194704724889129/-ext-10000/
+ pfile:/home/mwang/Projects/hive-git/build/ql/scratchdir/hive_2011-04-23_23-51-25_951_1689143600430803950/-ext-10000/
@@ -1280,7 +1282,8 @@
sum
-
+
+ org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator$Mode
FINAL
@@ -1316,7 +1319,8 @@
0.9
-
+
+ org.apache.hadoop.hive.ql.plan.GroupByDesc$Mode
MERGEPARTIAL
diff --git ql/src/test/results/compiler/plan/groupby2.q.xml ql/src/test/results/compiler/plan/groupby2.q.xml
index 400e86b..a172b4e 100755
--- ql/src/test/results/compiler/plan/groupby2.q.xml
+++ ql/src/test/results/compiler/plan/groupby2.q.xml
@@ -1,5 +1,5 @@
-
+
Stage-3
@@ -62,11 +62,11 @@
location
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
transient_lastDdlTime
- 1300721801
+ 1303627891
@@ -124,11 +124,11 @@
location
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
transient_lastDdlTime
- 1300721801
+ 1303627891
@@ -548,7 +548,8 @@
count
-
+
+ org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator$Mode
PARTIAL1
@@ -682,7 +683,8 @@
0.9
-
+
+ org.apache.hadoop.hive.ql.plan.GroupByDesc$Mode
HASH
@@ -1003,7 +1005,7 @@
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
src
@@ -1015,7 +1017,7 @@
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
src
@@ -1072,11 +1074,11 @@
location
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
transient_lastDdlTime
- 1300721801
+ 1303627891
@@ -1134,11 +1136,11 @@
location
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
transient_lastDdlTime
- 1300721801
+ 1303627891
@@ -1164,13 +1166,13 @@
- file:/tmp/krishnak/hive_2011-03-21_08-36-41_913_5459406223839387928/-ext-10001
+ file:/tmp/mwang/hive_2011-04-23_23-51-34_488_799887064116944445/-ext-10001
1
- file:/tmp/krishnak/hive_2011-03-21_08-36-41_913_5459406223839387928/-ext-10001/
+ file:/tmp/mwang/hive_2011-04-23_23-51-34_488_799887064116944445/-ext-10001/
@@ -1501,7 +1503,8 @@
count
-
+
+ org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator$Mode
COMPLETE
@@ -1530,7 +1533,8 @@
sum
-
+
+ org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator$Mode
FINAL
@@ -1566,7 +1570,8 @@
0.9
-
+
+ org.apache.hadoop.hive.ql.plan.GroupByDesc$Mode
MERGEPARTIAL
diff --git ql/src/test/results/compiler/plan/groupby3.q.xml ql/src/test/results/compiler/plan/groupby3.q.xml
index b9f115b..e21bd59 100644
--- ql/src/test/results/compiler/plan/groupby3.q.xml
+++ ql/src/test/results/compiler/plan/groupby3.q.xml
@@ -1,2024 +1,2029 @@
-
-
-
-
- Stage-3
-
-
-
-
-
- src
-
-
- org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
-
- org.apache.hadoop.mapred.TextInputFormat
-
-
- org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-
-
-
-
-
-
-
- name
- default.src
-
-
- columns.types
- string:string
-
-
- serialization.ddl
- struct src { string key, string value}
-
-
- serialization.format
- 1
-
-
- columns
- key,value
-
-
- bucket_count
- -1
-
-
- serialization.lib
- org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
-
- file.inputformat
- org.apache.hadoop.mapred.TextInputFormat
-
-
- file.outputformat
- org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-
-
- location
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
-
-
- transient_lastDdlTime
- 1300721804
-
-
-
-
- org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
-
-
-
- org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
-
- org.apache.hadoop.mapred.TextInputFormat
-
-
- org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-
-
-
-
- name
- default.src
-
-
- columns.types
- string:string
-
-
- serialization.ddl
- struct src { string key, string value}
-
-
- serialization.format
- 1
-
-
- columns
- key,value
-
-
- bucket_count
- -1
-
-
- serialization.lib
- org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
-
- file.inputformat
- org.apache.hadoop.mapred.TextInputFormat
-
-
- file.outputformat
- org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-
-
- location
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
-
-
- transient_lastDdlTime
- 1300721804
-
-
-
-
- org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
-
-
-
-
-
-
-
-
- src
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 0
-
-
-
-
-
-
-
-
-
-
- _col0
-
-
-
-
-
-
-
- string
-
-
-
-
-
-
-
-
-
-
- org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe
-
-
- org.apache.hadoop.mapred.SequenceFileInputFormat
-
-
- org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-
-
-
-
- columns
- _col0
-
-
- serialization.sort.order
- +
-
-
- columns.types
- uniontype<struct<_col0:string>>
-
-
-
-
-
-
- 1
-
-
-
-
- _col0
-
-
-
-
-
-
- _col0
-
-
- _col1
-
-
- _col2
-
-
- _col3
-
-
- _col4
-
-
-
-
-
-
-
- -1
-
-
-
-
-
-
- _col1
-
-
-
-
-
-
-
- double
-
-
-
-
-
-
-
-
- _col2
-
-
-
-
-
-
-
-
-
- count
-
-
- sum
-
-
-
-
-
-
-
-
- bigint
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- _col3
-
-
-
-
-
-
-
-
-
-
-
-
- _col4
-
-
-
-
-
-
-
-
-
-
-
-
- _col5
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
-
- org.apache.hadoop.mapred.SequenceFileInputFormat
-
-
- org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-
-
-
-
- columns
- _col0,_col1,_col2,_col3,_col4
-
-
- columns.types
- double,struct<count:bigint,sum:double>,struct<count:bigint,sum:double>,string,string
-
-
- escape.delim
- \
-
-
-
-
-
-
-
-
-
-
- CNTR_NAME_RS_55_NUM_INPUT_ROWS
-
-
- CNTR_NAME_RS_55_NUM_OUTPUT_ROWS
-
-
- CNTR_NAME_RS_55_TIME_TAKEN
-
-
- CNTR_NAME_RS_55_FATAL_ERROR
-
-
-
-
- RS_55
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- KEY._col0:0._col0
-
-
-
-
-
-
-
-
-
- VALUE._col0
-
-
-
-
-
-
-
-
-
- VALUE._col1
-
-
-
-
-
-
-
-
-
- VALUE._col2
-
-
-
-
-
-
-
-
-
- VALUE._col3
-
-
-
-
-
-
-
-
-
- VALUE._col4
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- _col0
-
-
-
-
-
-
- value
-
-
- src
-
-
-
-
-
-
-
-
-
-
-
- int
-
-
-
-
- 5
-
-
-
-
-
-
-
-
- org.apache.hadoop.hive.ql.udf.UDFSubstr
-
-
- substr
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- sum
-
-
-
- PARTIAL1
-
-
-
-
-
-
-
-
-
-
-
- value
-
-
- src
-
-
-
-
-
-
-
-
-
-
-
-
- 5
-
-
-
-
-
-
-
-
- org.apache.hadoop.hive.ql.udf.UDFSubstr
-
-
- substr
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- avg
-
-
-
-
-
-
-
-
-
-
-
-
-
- value
-
-
- src
-
-
-
-
-
-
-
-
-
-
-
-
- 5
-
-
-
-
-
-
-
-
- org.apache.hadoop.hive.ql.udf.UDFSubstr
-
-
- substr
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- true
-
-
-
-
-
- avg
-
-
-
-
-
-
-
-
-
-
-
-
-
- value
-
-
- src
-
-
-
-
-
-
-
-
-
-
-
-
- 5
-
-
-
-
-
-
-
-
- org.apache.hadoop.hive.ql.udf.UDFSubstr
-
-
- substr
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- max
-
-
-
-
-
-
-
-
-
-
-
-
-
- value
-
-
- src
-
-
-
-
-
-
-
-
-
-
-
-
- 5
-
-
-
-
-
-
-
-
- org.apache.hadoop.hive.ql.udf.UDFSubstr
-
-
- substr
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- min
-
-
-
-
-
-
-
-
-
-
-
-
-
- value
-
-
- src
-
-
-
-
-
-
-
-
-
-
-
-
- 5
-
-
-
-
-
-
-
-
- org.apache.hadoop.hive.ql.udf.UDFSubstr
-
-
- substr
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 0.5
-
-
-
-
-
-
-
-
-
- 0.9
-
-
-
- HASH
-
-
-
-
-
- _col0
-
-
- _col1
-
-
- _col2
-
-
- _col3
-
-
- _col4
-
-
- _col5
-
-
-
-
-
-
-
-
- CNTR_NAME_GBY_54_NUM_INPUT_ROWS
-
-
- CNTR_NAME_GBY_54_NUM_OUTPUT_ROWS
-
-
- CNTR_NAME_GBY_54_TIME_TAKEN
-
-
- CNTR_NAME_GBY_54_FATAL_ERROR
-
-
-
-
- GBY_54
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- _col0
-
-
-
-
-
-
-
-
-
-
-
-
- _col1
-
-
-
-
-
-
-
-
-
-
-
-
- _col2
-
-
-
-
-
-
-
-
-
-
-
-
- _col3
-
-
-
-
-
-
-
-
-
-
-
-
- _col4
-
-
-
-
-
-
-
-
-
-
-
-
- _col5
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- value
-
-
- src
-
-
-
-
-
-
-
-
-
-
-
- value
-
-
-
-
- true
-
-
-
-
-
-
- CNTR_NAME_SEL_53_NUM_INPUT_ROWS
-
-
- CNTR_NAME_SEL_53_NUM_OUTPUT_ROWS
-
-
- CNTR_NAME_SEL_53_TIME_TAKEN
-
-
- CNTR_NAME_SEL_53_FATAL_ERROR
-
-
-
-
- SEL_53
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- value
-
-
- src
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- src
-
-
-
-
-
-
-
-
-
- CNTR_NAME_TS_52_NUM_INPUT_ROWS
-
-
- CNTR_NAME_TS_52_NUM_OUTPUT_ROWS
-
-
- CNTR_NAME_TS_52_TIME_TAKEN
-
-
- CNTR_NAME_TS_52_FATAL_ERROR
-
-
-
-
-
-
- 1
-
-
-
-
- TS_52
-
-
-
-
-
-
-
-
- key
-
-
- src
-
-
-
-
-
-
-
-
-
-
-
-
- true
-
-
- BLOCK__OFFSET__INSIDE__FILE
-
-
- src
-
-
-
-
-
-
-
-
-
- true
-
-
- INPUT__FILE__NAME
-
-
- src
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 1
-
-
-
-
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
-
-
- src
-
-
-
-
-
-
-
-
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
-
-
- src
-
-
- org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
-
- org.apache.hadoop.mapred.TextInputFormat
-
-
- org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-
-
-
-
-
-
-
- name
- default.src
-
-
- columns.types
- string:string
-
-
- serialization.ddl
- struct src { string key, string value}
-
-
- serialization.format
- 1
-
-
- columns
- key,value
-
-
- bucket_count
- -1
-
-
- serialization.lib
- org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
-
- file.inputformat
- org.apache.hadoop.mapred.TextInputFormat
-
-
- file.outputformat
- org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-
-
- location
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
-
-
- transient_lastDdlTime
- 1300721804
-
-
-
-
- org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
-
-
-
- org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
-
- org.apache.hadoop.mapred.TextInputFormat
-
-
- org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-
-
-
-
- name
- default.src
-
-
- columns.types
- string:string
-
-
- serialization.ddl
- struct src { string key, string value}
-
-
- serialization.format
- 1
-
-
- columns
- key,value
-
-
- bucket_count
- -1
-
-
- serialization.lib
- org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
-
- file.inputformat
- org.apache.hadoop.mapred.TextInputFormat
-
-
- file.outputformat
- org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-
-
- location
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
-
-
- transient_lastDdlTime
- 1300721804
-
-
-
-
- org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- file:/tmp/krishnak/hive_2011-03-21_08-36-45_121_5129810727647323074/-ext-10001
-
-
- 1
-
-
- file:/tmp/krishnak/hive_2011-03-21_08-36-45_121_5129810727647323074/-ext-10001/
-
-
-
-
- org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
-
- org.apache.hadoop.mapred.TextInputFormat
-
-
- org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-
-
-
-
- columns
- _col0,_col1,_col2,_col3,_col4
-
-
- serialization.format
- 1
-
-
- columns.types
- double:double:double:string:string
-
-
-
-
-
-
- 1
-
-
-
-
-
-
- CNTR_NAME_FS_58_NUM_INPUT_ROWS
-
-
- CNTR_NAME_FS_58_NUM_OUTPUT_ROWS
-
-
- CNTR_NAME_FS_58_TIME_TAKEN
-
-
- CNTR_NAME_FS_58_FATAL_ERROR
-
-
-
-
- FS_58
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- _col0
-
-
-
-
-
-
-
-
-
-
-
-
- _col1
-
-
-
-
-
-
-
-
-
-
-
-
- _col2
-
-
-
-
-
-
-
-
-
-
-
-
- _col3
-
-
-
-
-
-
-
-
-
-
-
-
- _col4
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- _col4
-
-
- _col4
-
-
-
-
-
-
-
-
-
-
- _col3
-
-
- _col3
-
-
-
-
-
-
-
-
-
-
- _col2
-
-
- _col2
-
-
-
-
-
-
-
-
-
-
- _col1
-
-
- _col1
-
-
-
-
-
-
-
-
-
-
- _col0
-
-
- _col0
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- _col0
-
-
- _col1
-
-
- _col2
-
-
- _col3
-
-
- _col4
-
-
-
-
-
-
-
-
- CNTR_NAME_SEL_57_NUM_INPUT_ROWS
-
-
- CNTR_NAME_SEL_57_NUM_OUTPUT_ROWS
-
-
- CNTR_NAME_SEL_57_TIME_TAKEN
-
-
- CNTR_NAME_SEL_57_FATAL_ERROR
-
-
-
-
- SEL_57
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- _c0
-
-
- _col0
-
-
-
-
-
-
-
-
-
- _c1
-
-
- _col1
-
-
-
-
-
-
-
-
-
- _c2
-
-
- _col2
-
-
-
-
-
-
-
-
-
- _c3
-
-
- _col3
-
-
-
-
-
-
-
-
-
- _c4
-
-
- _col4
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- sum
-
-
-
- FINAL
-
-
-
-
-
-
-
- VALUE._col0
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- avg
-
-
-
-
-
-
-
-
-
- VALUE._col1
-
-
-
-
-
-
-
-
-
-
-
-
-
- true
-
-
-
-
-
- avg
-
-
-
- COMPLETE
-
-
-
-
-
-
-
- KEY._col0:0._col0
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- max
-
-
-
-
-
-
-
-
-
- VALUE._col3
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- min
-
-
-
-
-
-
-
-
-
- VALUE._col4
-
-
-
-
-
-
-
-
-
-
-
-
-
- 0.5
-
-
-
-
-
- 0.9
-
-
-
- MERGEPARTIAL
-
-
-
-
-
- _col0
-
-
- _col1
-
-
- _col2
-
-
- _col3
-
-
- _col4
-
-
-
-
-
-
-
-
- CNTR_NAME_GBY_56_NUM_INPUT_ROWS
-
-
- CNTR_NAME_GBY_56_NUM_OUTPUT_ROWS
-
-
- CNTR_NAME_GBY_56_TIME_TAKEN
-
-
- CNTR_NAME_GBY_56_FATAL_ERROR
-
-
-
-
- GBY_56
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- _col0
-
-
-
-
-
-
-
-
-
-
-
-
- _col1
-
-
-
-
-
-
-
-
-
-
-
-
- _col2
-
-
-
-
-
-
-
-
-
-
-
-
- _col3
-
-
-
-
-
-
-
-
-
-
-
-
- _col4
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+ Stage-3
+
+
+
+
+
+ src
+
+
+ org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+
+ org.apache.hadoop.mapred.TextInputFormat
+
+
+ org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+
+
+
+
+
+
+ name
+ default.src
+
+
+ columns.types
+ string:string
+
+
+ serialization.ddl
+ struct src { string key, string value}
+
+
+ serialization.format
+ 1
+
+
+ columns
+ key,value
+
+
+ bucket_count
+ -1
+
+
+ serialization.lib
+ org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+
+ file.inputformat
+ org.apache.hadoop.mapred.TextInputFormat
+
+
+ file.outputformat
+ org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+
+ location
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
+
+
+ transient_lastDdlTime
+ 1303627900
+
+
+
+
+ org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+
+
+
+ org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+
+ org.apache.hadoop.mapred.TextInputFormat
+
+
+ org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+
+
+
+ name
+ default.src
+
+
+ columns.types
+ string:string
+
+
+ serialization.ddl
+ struct src { string key, string value}
+
+
+ serialization.format
+ 1
+
+
+ columns
+ key,value
+
+
+ bucket_count
+ -1
+
+
+ serialization.lib
+ org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+
+ file.inputformat
+ org.apache.hadoop.mapred.TextInputFormat
+
+
+ file.outputformat
+ org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+
+ location
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
+
+
+ transient_lastDdlTime
+ 1303627900
+
+
+
+
+ org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+
+
+
+
+
+
+
+
+ src
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 0
+
+
+
+
+
+
+
+
+
+
+ _col0
+
+
+
+
+
+
+
+ string
+
+
+
+
+
+
+
+
+
+
+ org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe
+
+
+ org.apache.hadoop.mapred.SequenceFileInputFormat
+
+
+ org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+
+
+
+ columns
+ _col0
+
+
+ serialization.sort.order
+ +
+
+
+ columns.types
+ uniontype<struct<_col0:string>>
+
+
+
+
+
+
+ 1
+
+
+
+
+ _col0
+
+
+
+
+
+
+ _col0
+
+
+ _col1
+
+
+ _col2
+
+
+ _col3
+
+
+ _col4
+
+
+
+
+
+
+
+ -1
+
+
+
+
+
+
+ _col1
+
+
+
+
+
+
+
+ double
+
+
+
+
+
+
+
+
+ _col2
+
+
+
+
+
+
+
+
+
+ count
+
+
+ sum
+
+
+
+
+
+
+
+
+ bigint
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ _col3
+
+
+
+
+
+
+
+
+
+
+
+
+ _col4
+
+
+
+
+
+
+
+
+
+
+
+
+ _col5
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+
+ org.apache.hadoop.mapred.SequenceFileInputFormat
+
+
+ org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+
+
+
+ columns
+ _col0,_col1,_col2,_col3,_col4
+
+
+ columns.types
+ double,struct<count:bigint,sum:double>,struct<count:bigint,sum:double>,string,string
+
+
+ escape.delim
+ \
+
+
+
+
+
+
+
+
+
+
+ CNTR_NAME_RS_55_NUM_INPUT_ROWS
+
+
+ CNTR_NAME_RS_55_NUM_OUTPUT_ROWS
+
+
+ CNTR_NAME_RS_55_TIME_TAKEN
+
+
+ CNTR_NAME_RS_55_FATAL_ERROR
+
+
+
+
+ RS_55
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ KEY._col0:0._col0
+
+
+
+
+
+
+
+
+
+ VALUE._col0
+
+
+
+
+
+
+
+
+
+ VALUE._col1
+
+
+
+
+
+
+
+
+
+ VALUE._col2
+
+
+
+
+
+
+
+
+
+ VALUE._col3
+
+
+
+
+
+
+
+
+
+ VALUE._col4
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ _col0
+
+
+
+
+
+
+ value
+
+
+ src
+
+
+
+
+
+
+
+
+
+
+
+ int
+
+
+
+
+ 5
+
+
+
+
+
+
+
+
+ org.apache.hadoop.hive.ql.udf.UDFSubstr
+
+
+ substr
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ sum
+
+
+
+ org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator$Mode
+ PARTIAL1
+
+
+
+
+
+
+
+
+
+
+
+ value
+
+
+ src
+
+
+
+
+
+
+
+
+
+
+
+
+ 5
+
+
+
+
+
+
+
+
+ org.apache.hadoop.hive.ql.udf.UDFSubstr
+
+
+ substr
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ avg
+
+
+
+
+
+
+
+
+
+
+
+
+
+ value
+
+
+ src
+
+
+
+
+
+
+
+
+
+
+
+
+ 5
+
+
+
+
+
+
+
+
+ org.apache.hadoop.hive.ql.udf.UDFSubstr
+
+
+ substr
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ true
+
+
+
+
+
+ avg
+
+
+
+
+
+
+
+
+
+
+
+
+
+ value
+
+
+ src
+
+
+
+
+
+
+
+
+
+
+
+
+ 5
+
+
+
+
+
+
+
+
+ org.apache.hadoop.hive.ql.udf.UDFSubstr
+
+
+ substr
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ max
+
+
+
+
+
+
+
+
+
+
+
+
+
+ value
+
+
+ src
+
+
+
+
+
+
+
+
+
+
+
+
+ 5
+
+
+
+
+
+
+
+
+ org.apache.hadoop.hive.ql.udf.UDFSubstr
+
+
+ substr
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ min
+
+
+
+
+
+
+
+
+
+
+
+
+
+ value
+
+
+ src
+
+
+
+
+
+
+
+
+
+
+
+
+ 5
+
+
+
+
+
+
+
+
+ org.apache.hadoop.hive.ql.udf.UDFSubstr
+
+
+ substr
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 0.5
+
+
+
+
+
+
+
+
+
+ 0.9
+
+
+
+ org.apache.hadoop.hive.ql.plan.GroupByDesc$Mode
+ HASH
+
+
+
+
+
+ _col0
+
+
+ _col1
+
+
+ _col2
+
+
+ _col3
+
+
+ _col4
+
+
+ _col5
+
+
+
+
+
+
+
+
+ CNTR_NAME_GBY_54_NUM_INPUT_ROWS
+
+
+ CNTR_NAME_GBY_54_NUM_OUTPUT_ROWS
+
+
+ CNTR_NAME_GBY_54_TIME_TAKEN
+
+
+ CNTR_NAME_GBY_54_FATAL_ERROR
+
+
+
+
+ GBY_54
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ _col0
+
+
+
+
+
+
+
+
+
+
+
+
+ _col1
+
+
+
+
+
+
+
+
+
+
+
+
+ _col2
+
+
+
+
+
+
+
+
+
+
+
+
+ _col3
+
+
+
+
+
+
+
+
+
+
+
+
+ _col4
+
+
+
+
+
+
+
+
+
+
+
+
+ _col5
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ value
+
+
+ src
+
+
+
+
+
+
+
+
+
+
+
+ value
+
+
+
+
+ true
+
+
+
+
+
+
+ CNTR_NAME_SEL_53_NUM_INPUT_ROWS
+
+
+ CNTR_NAME_SEL_53_NUM_OUTPUT_ROWS
+
+
+ CNTR_NAME_SEL_53_TIME_TAKEN
+
+
+ CNTR_NAME_SEL_53_FATAL_ERROR
+
+
+
+
+ SEL_53
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ value
+
+
+ src
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ src
+
+
+
+
+
+
+
+
+
+ CNTR_NAME_TS_52_NUM_INPUT_ROWS
+
+
+ CNTR_NAME_TS_52_NUM_OUTPUT_ROWS
+
+
+ CNTR_NAME_TS_52_TIME_TAKEN
+
+
+ CNTR_NAME_TS_52_FATAL_ERROR
+
+
+
+
+
+
+ 1
+
+
+
+
+ TS_52
+
+
+
+
+
+
+
+
+ key
+
+
+ src
+
+
+
+
+
+
+
+
+
+
+
+
+ true
+
+
+ BLOCK__OFFSET__INSIDE__FILE
+
+
+ src
+
+
+
+
+
+
+
+
+
+ true
+
+
+ INPUT__FILE__NAME
+
+
+ src
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 1
+
+
+
+
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
+
+
+ src
+
+
+
+
+
+
+
+
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
+
+
+ src
+
+
+ org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+
+ org.apache.hadoop.mapred.TextInputFormat
+
+
+ org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+
+
+
+
+
+
+ name
+ default.src
+
+
+ columns.types
+ string:string
+
+
+ serialization.ddl
+ struct src { string key, string value}
+
+
+ serialization.format
+ 1
+
+
+ columns
+ key,value
+
+
+ bucket_count
+ -1
+
+
+ serialization.lib
+ org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+
+ file.inputformat
+ org.apache.hadoop.mapred.TextInputFormat
+
+
+ file.outputformat
+ org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+
+ location
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
+
+
+ transient_lastDdlTime
+ 1303627900
+
+
+
+
+ org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+
+
+
+ org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+
+ org.apache.hadoop.mapred.TextInputFormat
+
+
+ org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+
+
+
+ name
+ default.src
+
+
+ columns.types
+ string:string
+
+
+ serialization.ddl
+ struct src { string key, string value}
+
+
+ serialization.format
+ 1
+
+
+ columns
+ key,value
+
+
+ bucket_count
+ -1
+
+
+ serialization.lib
+ org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+
+ file.inputformat
+ org.apache.hadoop.mapred.TextInputFormat
+
+
+ file.outputformat
+ org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+
+ location
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
+
+
+ transient_lastDdlTime
+ 1303627900
+
+
+
+
+ org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ file:/tmp/mwang/hive_2011-04-23_23-51-42_686_7399638659402232159/-ext-10001
+
+
+ 1
+
+
+ file:/tmp/mwang/hive_2011-04-23_23-51-42_686_7399638659402232159/-ext-10001/
+
+
+
+
+ org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+
+ org.apache.hadoop.mapred.TextInputFormat
+
+
+ org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+
+
+
+ columns
+ _col0,_col1,_col2,_col3,_col4
+
+
+ serialization.format
+ 1
+
+
+ columns.types
+ double:double:double:string:string
+
+
+
+
+
+
+ 1
+
+
+
+
+
+
+ CNTR_NAME_FS_58_NUM_INPUT_ROWS
+
+
+ CNTR_NAME_FS_58_NUM_OUTPUT_ROWS
+
+
+ CNTR_NAME_FS_58_TIME_TAKEN
+
+
+ CNTR_NAME_FS_58_FATAL_ERROR
+
+
+
+
+ FS_58
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ _col0
+
+
+
+
+
+
+
+
+
+
+
+
+ _col1
+
+
+
+
+
+
+
+
+
+
+
+
+ _col2
+
+
+
+
+
+
+
+
+
+
+
+
+ _col3
+
+
+
+
+
+
+
+
+
+
+
+
+ _col4
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ _col4
+
+
+ _col4
+
+
+
+
+
+
+
+
+
+
+ _col3
+
+
+ _col3
+
+
+
+
+
+
+
+
+
+
+ _col2
+
+
+ _col2
+
+
+
+
+
+
+
+
+
+
+ _col1
+
+
+ _col1
+
+
+
+
+
+
+
+
+
+
+ _col0
+
+
+ _col0
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ _col0
+
+
+ _col1
+
+
+ _col2
+
+
+ _col3
+
+
+ _col4
+
+
+
+
+
+
+
+
+ CNTR_NAME_SEL_57_NUM_INPUT_ROWS
+
+
+ CNTR_NAME_SEL_57_NUM_OUTPUT_ROWS
+
+
+ CNTR_NAME_SEL_57_TIME_TAKEN
+
+
+ CNTR_NAME_SEL_57_FATAL_ERROR
+
+
+
+
+ SEL_57
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ _c0
+
+
+ _col0
+
+
+
+
+
+
+
+
+
+ _c1
+
+
+ _col1
+
+
+
+
+
+
+
+
+
+ _c2
+
+
+ _col2
+
+
+
+
+
+
+
+
+
+ _c3
+
+
+ _col3
+
+
+
+
+
+
+
+
+
+ _c4
+
+
+ _col4
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ sum
+
+
+
+ org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator$Mode
+ FINAL
+
+
+
+
+
+
+
+ VALUE._col0
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ avg
+
+
+
+
+
+
+
+
+
+ VALUE._col1
+
+
+
+
+
+
+
+
+
+
+
+
+
+ true
+
+
+
+
+
+ avg
+
+
+
+ org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator$Mode
+ COMPLETE
+
+
+
+
+
+
+
+ KEY._col0:0._col0
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ max
+
+
+
+
+
+
+
+
+
+ VALUE._col3
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ min
+
+
+
+
+
+
+
+
+
+ VALUE._col4
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 0.5
+
+
+
+
+
+ 0.9
+
+
+
+ org.apache.hadoop.hive.ql.plan.GroupByDesc$Mode
+ MERGEPARTIAL
+
+
+
+
+
+ _col0
+
+
+ _col1
+
+
+ _col2
+
+
+ _col3
+
+
+ _col4
+
+
+
+
+
+
+
+
+ CNTR_NAME_GBY_56_NUM_INPUT_ROWS
+
+
+ CNTR_NAME_GBY_56_NUM_OUTPUT_ROWS
+
+
+ CNTR_NAME_GBY_56_TIME_TAKEN
+
+
+ CNTR_NAME_GBY_56_FATAL_ERROR
+
+
+
+
+ GBY_56
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ _col0
+
+
+
+
+
+
+
+
+
+
+
+
+ _col1
+
+
+
+
+
+
+
+
+
+
+
+
+ _col2
+
+
+
+
+
+
+
+
+
+
+
+
+ _col3
+
+
+
+
+
+
+
+
+
+
+
+
+ _col4
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git ql/src/test/results/compiler/plan/groupby4.q.xml ql/src/test/results/compiler/plan/groupby4.q.xml
index a61c47e..6074237 100644
--- ql/src/test/results/compiler/plan/groupby4.q.xml
+++ ql/src/test/results/compiler/plan/groupby4.q.xml
@@ -1,5 +1,5 @@
-
+
Stage-3
@@ -62,11 +62,11 @@
location
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
transient_lastDdlTime
- 1300721807
+ 1303627908
@@ -124,11 +124,11 @@
location
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
transient_lastDdlTime
- 1300721807
+ 1303627908
@@ -406,7 +406,8 @@
0.9
-
+
+ org.apache.hadoop.hive.ql.plan.GroupByDesc$Mode
HASH
@@ -661,7 +662,7 @@
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
src
@@ -673,7 +674,7 @@
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
src
@@ -730,11 +731,11 @@
location
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
transient_lastDdlTime
- 1300721807
+ 1303627908
@@ -792,11 +793,11 @@
location
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
transient_lastDdlTime
- 1300721807
+ 1303627908
@@ -822,13 +823,13 @@
- file:/tmp/krishnak/hive_2011-03-21_08-36-48_370_3257228731841399758/-ext-10001
+ file:/tmp/mwang/hive_2011-04-23_23-51-49_762_3507990824520327443/-ext-10001
1
- file:/tmp/krishnak/hive_2011-03-21_08-36-48_370_3257228731841399758/-ext-10001/
+ file:/tmp/mwang/hive_2011-04-23_23-51-49_762_3507990824520327443/-ext-10001/
@@ -1036,7 +1037,8 @@
0.9
-
+
+ org.apache.hadoop.hive.ql.plan.GroupByDesc$Mode
MERGEPARTIAL
diff --git ql/src/test/results/compiler/plan/groupby5.q.xml ql/src/test/results/compiler/plan/groupby5.q.xml
index 04b9410..a77a7b8 100644
--- ql/src/test/results/compiler/plan/groupby5.q.xml
+++ ql/src/test/results/compiler/plan/groupby5.q.xml
@@ -1,5 +1,5 @@
-
+
Stage-3
@@ -62,11 +62,11 @@
location
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
transient_lastDdlTime
- 1300721810
+ 1303627915
@@ -124,11 +124,11 @@
location
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
transient_lastDdlTime
- 1300721810
+ 1303627915
@@ -387,7 +387,8 @@
sum
-
+
+ org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator$Mode
PARTIAL1
@@ -461,7 +462,8 @@
0.9
-
+
+ org.apache.hadoop.hive.ql.plan.GroupByDesc$Mode
HASH
@@ -754,7 +756,7 @@
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
src
@@ -766,7 +768,7 @@
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
src
@@ -823,11 +825,11 @@
location
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
transient_lastDdlTime
- 1300721810
+ 1303627915
@@ -885,11 +887,11 @@
location
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
transient_lastDdlTime
- 1300721810
+ 1303627915
@@ -915,13 +917,13 @@
- file:/tmp/krishnak/hive_2011-03-21_08-36-51_319_905974347288957661/-ext-10001
+ file:/tmp/mwang/hive_2011-04-23_23-51-57_065_3585524958054091286/-ext-10001
1
- file:/tmp/krishnak/hive_2011-03-21_08-36-51_319_905974347288957661/-ext-10001/
+ file:/tmp/mwang/hive_2011-04-23_23-51-57_065_3585524958054091286/-ext-10001/
@@ -1169,7 +1171,8 @@
sum
-
+
+ org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator$Mode
FINAL
@@ -1205,7 +1208,8 @@
0.9
-
+
+ org.apache.hadoop.hive.ql.plan.GroupByDesc$Mode
MERGEPARTIAL
diff --git ql/src/test/results/compiler/plan/groupby6.q.xml ql/src/test/results/compiler/plan/groupby6.q.xml
index 7011f45..5099f1b 100644
--- ql/src/test/results/compiler/plan/groupby6.q.xml
+++ ql/src/test/results/compiler/plan/groupby6.q.xml
@@ -1,5 +1,5 @@
-
+
Stage-3
@@ -62,11 +62,11 @@
location
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
transient_lastDdlTime
- 1300721813
+ 1303627922
@@ -124,11 +124,11 @@
location
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
transient_lastDdlTime
- 1300721813
+ 1303627922
@@ -406,7 +406,8 @@
0.9
-
+
+ org.apache.hadoop.hive.ql.plan.GroupByDesc$Mode
HASH
@@ -661,7 +662,7 @@
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
src
@@ -673,7 +674,7 @@
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
src
@@ -730,11 +731,11 @@
location
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
transient_lastDdlTime
- 1300721813
+ 1303627922
@@ -792,11 +793,11 @@
location
- pfile:/home/krishnak/hive-howl/hive-git-apache/build/ql/test/data/warehouse/src
+ pfile:/home/mwang/Projects/hive-git/build/ql/test/data/warehouse/src
transient_lastDdlTime
- 1300721813
+ 1303627922
@@ -822,13 +823,13 @@
- file:/tmp/krishnak/hive_2011-03-21_08-36-54_291_8561242536647244425/-ext-10001
+ file:/tmp/mwang/hive_2011-04-23_23-52-03_964_6361576996846647043/-ext-10001
1
- file:/tmp/krishnak/hive_2011-03-21_08-36-54_291_8561242536647244425/-ext-10001/
+ file:/tmp/mwang/hive_2011-04-23_23-52-03_964_6361576996846647043/-ext-10001/
@@ -1036,7 +1037,8 @@
0.9
-
+
+ org.apache.hadoop.hive.ql.plan.GroupByDesc$Mode
MERGEPARTIAL