From b9c2a3ee0218228f0ca39b8cabec5e3a5db67ba2 Mon Sep 17 00:00:00 2001 From: Syed Albiz Date: Sat, 30 Jul 2011 13:34:26 -0700 Subject: [PATCH 1/1] fix the shit that broke diff --git ql/src/java/org/apache/hadoop/hive/ql/index/TableBasedIndexHandler.java ql/src/java/org/apache/hadoop/hive/ql/index/TableBasedIndexHandler.java index 02ab78c..06b3a2c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/index/TableBasedIndexHandler.java +++ ql/src/java/org/apache/hadoop/hive/ql/index/TableBasedIndexHandler.java @@ -66,6 +66,9 @@ public abstract class TableBasedIndexHandler extends AbstractIndexHandler { new PartitionDesc(desc, null), indexTbl.getTableName(), new PartitionDesc(Utilities.getTableDesc(baseTbl), null), baseTbl.getTableName(), indexTbl.getDbName()); + if (indexBuilder == null) { + throw new HiveException("Cannot construct index without map-side aggregation"); + } indexBuilderTasks.add(indexBuilder); } else { @@ -89,6 +92,9 @@ public abstract class TableBasedIndexHandler extends AbstractIndexHandler { Task indexBuilder = getIndexBuilderMapRedTask(inputs, outputs, index.getSd().getCols(), true, new PartitionDesc(indexPart), indexTbl.getTableName(), new PartitionDesc(basePart), baseTbl.getTableName(), indexTbl.getDbName()); + if (indexBuilder == null) { + throw new HiveException("Cannot construct index without map-side aggregation"); + } indexBuilderTasks.add(indexBuilder); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java index 61bbbf5..5090c97 100644 --- ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java +++ ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java @@ -280,9 +280,9 @@ public class BitmapIndexHandler extends TableBasedIndexHandler { } // Require clusterby ROWOFFSET if map-size aggregation is off. + // TODO: Make this work wihtout map side aggregation if (!conf.get("hive.map.aggr", null).equals("true")) { - command.append(" CLUSTER BY "); - command.append(VirtualColumn.ROWOFFSET.getName()); + return null; } Driver driver = new Driver(conf); diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEWAHBitmap.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEWAHBitmap.java index 0a3df09..d896b8e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEWAHBitmap.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEWAHBitmap.java @@ -92,14 +92,19 @@ public class GenericUDAFEWAHBitmap extends AbstractGenericUDAFResolver { inputOI = (PrimitiveObjectInspector) parameters[0]; return ObjectInspectorFactory .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableLongObjectInspector); - } else { - //no map aggregation + } else if (m == Mode.PARTIAL2 || m == Mode.FINAL) { internalMergeOI = (StandardListObjectInspector) parameters[0]; bitmapLongOI = PrimitiveObjectInspectorFactory.writableLongObjectInspector; inputOI = PrimitiveObjectInspectorFactory.writableByteObjectInspector; loi = (StandardListObjectInspector) ObjectInspectorFactory .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableLongObjectInspector); return loi; + } else { // Mode.COMPLETE, ie. no map-side aggregation, requires ordering + bitmapLongOI = PrimitiveObjectInspectorFactory.writableLongObjectInspector; + inputOI = PrimitiveObjectInspectorFactory.writableByteObjectInspector; + loi = (StandardListObjectInspector) ObjectInspectorFactory + .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableLongObjectInspector); + return loi; } } -- 1.7.4.4