Index: ql/src/gen-javabean/org/apache/hadoop/hive/ql/plan/api/OperatorType.java =================================================================== --- ql/src/gen-javabean/org/apache/hadoop/hive/ql/plan/api/OperatorType.java (revision 997835) +++ ql/src/gen-javabean/org/apache/hadoop/hive/ql/plan/api/OperatorType.java (working copy) @@ -6,12 +6,10 @@ package org.apache.hadoop.hive.ql.plan.api; -import java.util.Set; -import java.util.HashSet; -import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + import org.apache.thrift.IntRangeSet; -import java.util.Map; -import java.util.HashMap; public class OperatorType { public static final int JOIN = 0; @@ -30,24 +28,28 @@ public static final int UDTF = 13; public static final int LATERALVIEWJOIN = 14; public static final int LATERALVIEWFORWARD = 15; + public static final int JDBMDUMMY = 16; + public static final int JDBMSINK = 17; public static final IntRangeSet VALID_VALUES = new IntRangeSet( - JOIN, - MAPJOIN, - EXTRACT, - FILTER, - FORWARD, - GROUPBY, - LIMIT, - SCRIPT, - SELECT, - TABLESCAN, - FILESINK, - REDUCESINK, - UNION, - UDTF, - LATERALVIEWJOIN, - LATERALVIEWFORWARD ); + JOIN, + MAPJOIN, + EXTRACT, + FILTER, + FORWARD, + GROUPBY, + LIMIT, + SCRIPT, + SELECT, + TABLESCAN, + FILESINK, + REDUCESINK, + UNION, + UDTF, + LATERALVIEWJOIN, + LATERALVIEWFORWARD, + JDBMDUMMY, + JDBMSINK); public static final Map VALUES_TO_NAMES = new HashMap() {{ put(JOIN, "JOIN"); @@ -66,5 +68,7 @@ put(UDTF, "UDTF"); put(LATERALVIEWJOIN, "LATERALVIEWJOIN"); put(LATERALVIEWFORWARD, "LATERALVIEWFORWARD"); + put(JDBMDUMMY, "JDBMDUMMY"); + put(JDBMSINK, "JDBMSINK"); }}; } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java (revision 997835) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java (working copy) @@ -21,19 +21,24 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.persistence.RowContainer; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.MapJoinDesc; import org.apache.hadoop.hive.ql.plan.api.OperatorType; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; public abstract class AbstractMapJoinOperator extends CommonJoinOperator implements @@ -119,7 +124,241 @@ } initializeChildren(hconf); } + private HashMap> getObjectInspectors( + Map> exprEntries, + ObjectInspector[] inputObjInspector, + boolean smallTables) throws HiveException { + HashMap> result = new HashMap>(); + for (Entry> exprEntry : exprEntries + .entrySet()) { + Byte alias = exprEntry.getKey(); + + //get big table + if(alias == (byte) posBigTable && smallTables){ + //skip the big tables + continue; + } + + List exprList = exprEntry.getValue(); + ArrayList fieldOIList = new ArrayList(); + for (int i = 0; i < exprList.size(); i++) { + fieldOIList.add(exprList.get(i).initialize(inputObjInspector[alias])); + } + result.put(alias, fieldOIList); + } + return result; + } + + private HashMap> getStandardObjectInspectors( + Map> aliasToObjectInspectors,boolean smallTables) { + HashMap> result = new HashMap>(); + for (Entry> oiEntry : aliasToObjectInspectors + .entrySet()) { + Byte alias = oiEntry.getKey(); + + //get big table + if(alias == (byte) posBigTable && smallTables){ + //skip the big tables + continue; + } + + List oiList = oiEntry.getValue(); + ArrayList fieldOIList = new ArrayList( + oiList.size()); + for (int i = 0; i < oiList.size(); i++) { + fieldOIList.add(ObjectInspectorUtils.getStandardObjectInspector(oiList + .get(i), ObjectInspectorCopyOption.WRITABLE)); + } + result.put(alias, fieldOIList); + } + return result; + + } + protected int populateJoinKeyValue(Map> outMap, + Map> inputMap,boolean smallTables) { + + int total = 0; + + Iterator>> entryIter = inputMap + .entrySet().iterator(); + while (entryIter.hasNext()) { + Map.Entry> e = entryIter.next(); + Byte key = order[e.getKey()]; + + //get big table + if(key == (byte) posBigTable && smallTables){ + //skip the big tables + continue; + } + + + List expr = e.getValue(); + int sz = expr.size(); + total += sz; + + List valueFields = new ArrayList(); + + for (int j = 0; j < sz; j++) { + valueFields.add(ExprNodeEvaluatorFactory.get(expr.get(j))); + } + + outMap.put(key, valueFields); + } + + return total; + } + + protected void initializeOp(Configuration hconf, boolean smallTablesOnly) throws HiveException { + + numMapRowsRead = 0; + firstRow = true; + + //for small tables only; so get the big table position first + posBigTable = conf.getPosBigTable(); + if (order == null) { + order = conf.getTagOrder(); + } + //initialize some variables, which used to be initialized in CommonJoinOperator + condn = conf.getConds(); + numAliases = conf.getExprs().size(); + this.hconf = hconf; + totalSz = 0; + + + //important to adjust the inputObjInspectors + //the index for inputObjInspectors is based on parentID + //and parent ID matches with alias of tables; + //since we change the parent for mapjoin operation, + //we should leave the position for the big table + /* + if(smallTablesOnly){ + int last_position = inputObjInspectors.length; + inputObjInspectors = Arrays.copyOf(inputObjInspectors, inputObjInspectors.length+1); + + + for(int i = last_position; i > posBigTable;i--){ + inputObjInspectors[i]=inputObjInspectors[i-1]; + } + }else{ + if(posBigTable > 0){ + inputObjInspectors = Arrays.copyOf(inputObjInspectors, posBigTable+1); + inputObjInspectors[posBigTable]=inputObjInspectors[0]; + } + }*/ + + //process join keys + joinKeys = new HashMap>(); + populateJoinKeyValue(joinKeys, conf.getKeys(),smallTablesOnly); + joinKeysObjectInspectors = getObjectInspectors(joinKeys, + inputObjInspectors,smallTablesOnly); + joinKeysStandardObjectInspectors = getStandardObjectInspectors(joinKeysObjectInspectors,smallTablesOnly); + + + //process join values + joinValues = new HashMap>(); + populateJoinKeyValue(joinValues, conf.getExprs(),smallTablesOnly); + joinValuesObjectInspectors = getObjectInspectors(joinValues,inputObjInspectors,smallTablesOnly); + joinValuesStandardObjectInspectors = getStandardObjectInspectors(joinValuesObjectInspectors,smallTablesOnly); + + +/* + if(!smallTablesOnly){ + this.handleSkewJoin = conf.getHandleSkewJoin(); + + heartbeatInterval = HiveConf.getIntVar(hconf, + HiveConf.ConfVars.HIVESENDHEARTBEAT); + countAfterReport = 0; + + totalSz = 0; + // Map that contains the rows for each alias + storage = new HashMap>>(); + + joinValues = new HashMap>(); + + if (order == null) { + order = conf.getTagOrder(); + } + condn = conf.getConds(); + noOuterJoin = conf.isNoOuterJoin(); + + totalSz = populateJoinKeyValue(joinValues, conf.getExprs()); + + dummyObj = new Object[numAliases]; + dummyObjVectors = new RowContainer[numAliases]; + + joinEmitInterval = HiveConf.getIntVar(hconf, + HiveConf.ConfVars.HIVEJOINEMITINTERVAL); + joinCacheSize = HiveConf.getIntVar(hconf, + HiveConf.ConfVars.HIVEJOINCACHESIZE); + + // construct dummy null row (indicating empty table) and + // construct spill table serde which is used if input is too + // large to fit into main memory. + byte pos = 0; + for (Byte alias : order) { + int sz = conf.getExprs().get(alias).size(); + ArrayList nr = new ArrayList(sz); + + for (int j = 0; j < sz; j++) { + nr.add(null); + } + dummyObj[pos] = nr; + // there should be only 1 dummy object in the RowContainer + RowContainer> values = getRowContainer(hconf, pos, + alias, 1); + values.add((ArrayList) dummyObj[pos]); + dummyObjVectors[pos] = values; + + // if serde is null, the input doesn't need to be spilled out + // e.g., the output columns does not contains the input table + RowContainer rc = getRowContainer(hconf, pos, alias, joinCacheSize); + storage.put(pos, rc); + + pos++; + } + + forwardCache = new Object[totalSz]; + + outputObjInspector = getJoinOutputObjectInspector(order, + joinValuesStandardObjectInspectors, conf); + LOG.info("JOIN " + + ((StructObjectInspector) outputObjInspector).getTypeName() + + " totalsz = " + totalSz); + + //from AbstractMapJoinOperator initializeOP + emptyList = new RowContainer>(1, hconf); + RowContainer bigPosRC = getRowContainer(hconf, (byte) posBigTable, + order[posBigTable], joinCacheSize); + storage.put((byte) posBigTable, bigPosRC); + + mapJoinRowsKey = HiveConf.getIntVar(hconf, + HiveConf.ConfVars.HIVEMAPJOINROWSIZE); + + List structFields = ((StructObjectInspector) outputObjInspector) + .getAllStructFieldRefs(); + if (conf.getOutputColumnNames().size() < structFields.size()) { + List structFieldObjectInspectors = new ArrayList(); + for (Byte alias : order) { + int sz = conf.getExprs().get(alias).size(); + List retained = conf.getRetainList().get(alias); + for (int i = 0; i < sz; i++) { + int position = retained.get(i); + structFieldObjectInspectors.add(structFields.get(position) + .getFieldObjectInspector()); + } + } + outputObjInspector = ObjectInspectorFactory + .getStandardStructObjectInspector(conf.getOutputColumnNames(), + structFieldObjectInspectors); + } + initializeChildren(hconf); + + } +*/ + } + + @Override protected void fatalErrorMessage(StringBuilder errMsg, long counterCode) { errMsg.append("Operator " + getOperatorId() + " (id=" + id + "): " Index: ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java (revision 997835) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java (working copy) @@ -159,6 +159,7 @@ this.storage = clone.storage; this.condn = clone.condn; + this.conf = clone.getConf(); this.setSchema(clone.getSchema()); this.alias = clone.alias; @@ -212,6 +213,8 @@ return total; } + + protected static HashMap> getObjectInspectorsFromEvaluators( Map> exprEntries, ObjectInspector[] inputObjInspector) throws HiveException { @@ -219,6 +222,7 @@ for (Entry> exprEntry : exprEntries .entrySet()) { Byte alias = exprEntry.getKey(); + List exprList = exprEntry.getValue(); ArrayList fieldOIList = new ArrayList(); for (int i = 0; i < exprList.size(); i++) { @@ -265,6 +269,7 @@ Configuration hconf; + @Override protected void initializeOp(Configuration hconf) throws HiveException { this.handleSkewJoin = conf.getHandleSkewJoin(); Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (revision 997835) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (working copy) @@ -42,6 +42,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.filecache.DistributedCache; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; @@ -465,12 +467,12 @@ */ @Override public int execute(DriverContext driverContext) { - + IOPrepareCache ioPrepareCache = IOPrepareCache.get(); ioPrepareCache.clear(); - + boolean success = true; - + String invalidReason = work.isInvalid(); if (invalidReason != null) { throw new RuntimeException("Plan invalid, Reason: " + invalidReason); @@ -586,8 +588,28 @@ HiveConf.setVar(job, HiveConf.ConfVars.HADOOPJOBNAME, "JOB" + Utilities.randGen.nextInt()); } + try { + //propagate the file to distributed cache + MapredLocalWork localwork =work.getMapLocalWork(); + if(localwork != null){ + boolean localMode = HiveConf.getVar(job, HiveConf.ConfVars.HADOOPJT).equals("local"); + if(!localMode){ + LOG.info("=========Adding files to distributed cache================"); + Path hdfsPath = new Path(localwork.getTmpFileURI()); + FileSystem fs = hdfsPath.getFileSystem(job); + FileStatus[] jdbmFiles = fs.listStatus(hdfsPath); + for(int i =0; i> dummyOps = localWork.getDummyParentOp(); + for(Operator dummyOp : dummyOps){ + dummyOp.setExecContext(execContext); + dummyOp.initialize(jc,null); + } + + + + + if( localWork.getAliasToFetchWork() == null){ + // no need to initialize this local work any more + return; + } fetchOperators = new HashMap(); Map fetchOpJobConfMap = new HashMap(); @@ -147,6 +164,7 @@ this.execContext.setFetchOperators(fetchOperators); // defer processing of map local operators to first row if in case there // is no input (??) + } catch (Throwable e) { abort = true; if (e instanceof OutOfMemoryError) { @@ -228,6 +246,16 @@ // ideally hadoop should let us know whether map execution failed or not try { mo.close(abort); + + //for close the local work + if(localWork != null){ + List> dummyOps = localWork.getDummyParentOp(); + + for(Operator dummyOp : dummyOps){ + dummyOp.close(abort); + } + } + if (fetchOperators != null) { MapredLocalWork localWork = mo.getConf().getMapLocalWork(); for (Map.Entry entry : fetchOperators.entrySet()) { Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapperContext.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapperContext.java (revision 997835) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapperContext.java (working copy) @@ -35,7 +35,7 @@ private JobConf jc; private IOContext ioCxt; - + public ExecMapperContext() { ioCxt = IOContext.get(); } @@ -194,7 +194,7 @@ public void setFetchOperators(Map fetchOperators) { this.fetchOperators = fetchOperators; } - + public IOContext getIoCxt() { return ioCxt; } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java (revision 997835) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java (working copy) @@ -49,10 +49,10 @@ import org.apache.hadoop.hive.serde2.Serializer; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.SubStructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.LongWritable; Index: ql/src/java/org/apache/hadoop/hive/ql/exec/JDBMDummyOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/JDBMDummyOperator.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/JDBMDummyOperator.java (revision 0) @@ -0,0 +1,58 @@ +package org.apache.hadoop.hive.ql.exec; + +import java.io.Serializable; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.plan.JDBMDummyDesc; +import org.apache.hadoop.hive.ql.plan.TableDesc; +import org.apache.hadoop.hive.ql.plan.api.OperatorType; +import org.apache.hadoop.hive.serde2.Deserializer; + +public class JDBMDummyOperator extends Operator implements Serializable{ + + @Override + protected void initializeOp(Configuration hconf) throws HiveException { + + + TableDesc tbl = this.getConf().getTbl(); + try{ + Deserializer serde = tbl.getDeserializerClass().newInstance(); + serde.initialize(hconf, tbl.getProperties()); + + + this.outputObjInspector = serde.getObjectInspector(); + initializeChildren(hconf); + }catch(Exception e){ + LOG.error("Generating output obj inspector from dummy object error"); + e.printStackTrace(); + } + + } + + + + + @Override + public void processOp(Object row, int tag) throws HiveException{ + + } + + @Override + /* + * Flush the hashtable into jdbm file + * Load this jdbm file into HDFS only + */ + public void closeOp(boolean abort) throws HiveException{ + } + @Override + public String getName() { + return "JDBMDUMMY"; + } + + @Override + public int getType() { + return OperatorType.JDBMDUMMY; + } + +} Property changes on: ql/src/java/org/apache/hadoop/hive/ql/exec/JDBMDummyOperator.java ___________________________________________________________________ Added: svn:executable + * Index: ql/src/java/org/apache/hadoop/hive/ql/exec/JDBMSinkOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/JDBMSinkOperator.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/JDBMSinkOperator.java (revision 0) @@ -0,0 +1,639 @@ +package org.apache.hadoop.hive.ql.exec; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.exec.persistence.HashMapWrapper; +import org.apache.hadoop.hive.ql.exec.persistence.MapJoinObjectKey; +import org.apache.hadoop.hive.ql.exec.persistence.MapJoinObjectValue; +import org.apache.hadoop.hive.ql.exec.persistence.RowContainer; +import org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.MapJoinDesc; +import org.apache.hadoop.hive.ql.plan.TableDesc; +import org.apache.hadoop.hive.ql.plan.api.OperatorType; +import org.apache.hadoop.hive.serde2.SerDe; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; +import org.apache.hadoop.mapred.SequenceFileInputFormat; +import org.apache.hadoop.util.ReflectionUtils; + + +public class JDBMSinkOperator extends Operator implements Serializable { + private static final long serialVersionUID = 1L; + private static final Log LOG = LogFactory.getLog(JDBMSinkOperator.class + .getName()); + + //from abstract map join operator + /** + * The expressions for join inputs's join keys. + */ + protected transient Map> joinKeys; + /** + * The ObjectInspectors for the join inputs's join keys. + */ + protected transient Map> joinKeysObjectInspectors; + /** + * The standard ObjectInspectors for the join inputs's join keys. + */ + protected transient Map> joinKeysStandardObjectInspectors; + + protected transient int posBigTableTag = -1; // one of the tables that is not in memory + protected transient int posBigTableAlias = -1; // one of the tables that is not in memory + transient int mapJoinRowsKey; // rows for a given key + + protected transient RowContainer> emptyList = null; + + transient int numMapRowsRead; + protected transient int totalSz; // total size of the composite object + transient boolean firstRow; + private boolean smallTablesOnly; + + //from common join operator + protected transient int numAliases; // number of aliases + /** + * The expressions for join outputs. + */ + protected transient Map> joinValues; + /** + * The ObjectInspectors for the join inputs. + */ + protected transient Map> joinValuesObjectInspectors; + /** + * The standard ObjectInspectors for the join inputs. + */ + protected transient Map> joinValuesStandardObjectInspectors; + + protected transient Byte[] order; // order in which the results should + Configuration hconf; + protected transient Byte alias; + protected transient Map spillTableDesc; // spill tables are + + //from map join operator + protected transient Map> mapJoinTables; + //static transient Map mapMetadata = new HashMap(); + //private MapMetaData mapMetadata; + + + + + public static class JDBMSinkObjectCtx { + ObjectInspector standardOI; + SerDe serde; + TableDesc tblDesc; + Configuration conf; + + /** + * @param standardOI + * @param serde + */ + public JDBMSinkObjectCtx(ObjectInspector standardOI, SerDe serde, + TableDesc tblDesc, Configuration conf) { + this.standardOI = standardOI; + this.serde = serde; + this.tblDesc = tblDesc; + this.conf = conf; + } + + /** + * @return the standardOI + */ + public ObjectInspector getStandardOI() { + return standardOI; + } + + /** + * @return the serde + */ + public SerDe getSerDe() { + return serde; + } + + public TableDesc getTblDesc() { + return tblDesc; + } + + public Configuration getConf() { + return conf; + } + + } + + + private static final transient String[] FATAL_ERR_MSG = { + null, // counter value 0 means no error + "Mapside join size exceeds hive.mapjoin.maxsize. " + + "Please increase that or remove the mapjoin hint." + }; + transient int metadataKeyTag; + transient int[] metadataValueTag; + transient int maxMapJoinSize; + + + + public JDBMSinkOperator(){ + //super(); + } + + public JDBMSinkOperator(MapJoinOperator mjop){ + this.conf = mjop.getConf(); + } + + + + + @Override + protected void initializeOp(Configuration hconf) throws HiveException { + //super.initializeOp(hconf); + + //mapMetaData = new MapMetaData(); + maxMapJoinSize = HiveConf.getIntVar(hconf, + HiveConf.ConfVars.HIVEMAXMAPJOINSIZE); + + + + numMapRowsRead = 0; + firstRow = true; + + //for small tables only; so get the big table position first + posBigTableTag = conf.getPosBigTable(); + + order = conf.getTagOrder(); + + + posBigTableAlias=order[posBigTableTag]; + /* + int length = inputObjInspectors.length; + if(posBigTableAlias < length){ + inputObjInspectors = Arrays.copyOf(inputObjInspectors, length+1); + for(int i=length; i > posBigTableAlias; i--){ + inputObjInspectors[i]=inputObjInspectors[i-1]; + } + }*/ + + + + + //initialize some variables, which used to be initialized in CommonJoinOperator + + numAliases = conf.getExprs().size(); + this.hconf = hconf; + totalSz = 0; + + + //process join keys + joinKeys = new HashMap>(); + populateJoinKeyValue(joinKeys, conf.getKeys()); + joinKeysObjectInspectors = getObjectInspectors(joinKeys,inputObjInspectors); + joinKeysStandardObjectInspectors = getStandardObjectInspectors(joinKeysObjectInspectors); + + + //process join values + joinValues = new HashMap>(); + populateJoinKeyValue(joinValues, conf.getExprs()); + joinValuesObjectInspectors = getObjectInspectors(joinValues,inputObjInspectors); + joinValuesStandardObjectInspectors = getStandardObjectInspectors(joinValuesObjectInspectors); + + + + metadataValueTag = new int[numAliases]; + for (int pos = 0; pos < numAliases; pos++) { + metadataValueTag[pos] = -1; + } + + mapJoinTables = new HashMap>(); + + // initialize the hash tables for other tables + for (Byte pos:order) { + if (pos == posBigTableTag) { + continue; + } + + int cacheSize = HiveConf.getIntVar(hconf, + HiveConf.ConfVars.HIVEMAPJOINCACHEROWS); + HashMapWrapper hashTable = new HashMapWrapper( + cacheSize); + + mapJoinTables.put(pos, hashTable); + } + } + + private HashMap> getObjectInspectors( + Map> exprEntries, + ObjectInspector[] inputObjInspector) throws HiveException { + HashMap> result = new HashMap>(); + for (Entry> exprEntry : exprEntries + .entrySet()) { + Byte alias = exprEntry.getKey(); + + //get big table + if(alias == (byte) posBigTableAlias){ + //skip the big tables + continue; + } + + List exprList = exprEntry.getValue(); + ArrayList fieldOIList = new ArrayList(); + for (int i = 0; i < exprList.size(); i++) { + fieldOIList.add(exprList.get(i).initialize(inputObjInspector[alias])); + } + result.put(alias, fieldOIList); + } + return result; + } + + + private HashMap> getStandardObjectInspectors( + Map> aliasToObjectInspectors) { + HashMap> result = new HashMap>(); + for (Entry> oiEntry : aliasToObjectInspectors + .entrySet()) { + Byte alias = oiEntry.getKey(); + + //get big table + if(alias == (byte) posBigTableAlias ){ + //skip the big tables + continue; + } + + List oiList = oiEntry.getValue(); + ArrayList fieldOIList = new ArrayList( + oiList.size()); + for (int i = 0; i < oiList.size(); i++) { + fieldOIList.add(ObjectInspectorUtils.getStandardObjectInspector(oiList + .get(i), ObjectInspectorCopyOption.WRITABLE)); + } + result.put(alias, fieldOIList); + } + return result; + + } + protected int populateJoinKeyValue(Map> outMap, + Map> inputMap) { + + int total = 0; + + Iterator>> entryIter = inputMap + .entrySet().iterator(); + while (entryIter.hasNext()) { + Map.Entry> e = entryIter.next(); + Byte key = order[e.getKey()]; + + List valueFields = new ArrayList(); + + List expr = e.getValue(); + int sz = expr.size(); + total += sz; + + for (int j = 0; j < sz; j++) { + if(key == (byte) posBigTableAlias){ + valueFields.add(null); + }else{ + valueFields.add(ExprNodeEvaluatorFactory.get(expr.get(j))); + } + } + + outMap.put(key, valueFields); + } + + return total; + } + + /* + * This operator only process small tables + * Read the key/value pairs + * Load them into hashtable + */ + @Override + public void processOp(Object row, int tag) throws HiveException{ + //let the mapJoinOp process these small tables + try{ + alias = order[tag]; + //alias = (byte)tag; + + // compute keys and values as StandardObjects + ArrayList key = computeValues(row, joinKeys.get(alias), + joinKeysObjectInspectors.get(alias)); + ArrayList value = computeValues(row, joinValues.get(alias), + joinValuesObjectInspectors.get(alias)); + + // does this source need to be stored in the hash map + + if (firstRow) { + //metadatakeytag is always 0 + //metadataKeyTag = nextVal++; + metadataKeyTag = -1; + + TableDesc keyTableDesc = conf.getKeyTblDesc(); + SerDe keySerializer = (SerDe) ReflectionUtils.newInstance( + keyTableDesc.getDeserializerClass(), null); + keySerializer.initialize(null, keyTableDesc.getProperties()); + + MapJoinMetaData.put(Integer.valueOf(metadataKeyTag), + new JDBMSinkObjectCtx( + ObjectInspectorUtils + .getStandardObjectInspector(keySerializer + .getObjectInspector(), + ObjectInspectorCopyOption.WRITABLE), keySerializer, + keyTableDesc, hconf)); + + + firstRow = false; + } + + //reportProgress(); + numMapRowsRead++; + + if ((numMapRowsRead > maxMapJoinSize) && (reporter != null) + && (counterNameToEnum != null)) { + // update counter + LOG + .warn("Too many rows in map join tables. Fatal error counter will be incremented!!"); + incrCounter(fatalErrorCntr, 1); + fatalError = true; + return; + } + + HashMapWrapper hashTable = mapJoinTables.get((byte) tag); + MapJoinObjectKey keyMap = new MapJoinObjectKey(metadataKeyTag, key); + MapJoinObjectValue o = hashTable.get(keyMap); + RowContainer res = null; + + boolean needNewKey = true; + if (o == null) { + int bucketSize = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEMAPJOINBUCKETCACHESIZE); + res = getRowContainer(hconf, (byte) tag, order[tag], bucketSize); + res.add(value); + } else { + res = o.getObj(); + res.add(value); + // If key already exists, HashMapWrapper.get() guarantees it is + // already in main memory HashMap + // cache. So just replacing the object value should update the + // HashMapWrapper. This will save + // the cost of constructing the new key/object and deleting old one + // and inserting the new one. + if (hashTable.cacheSize() > 0) { + o.setObj(res); + needNewKey = false; + } + } + + if (metadataValueTag[tag] == -1) { + //metadataValueTag[tag] = nextVal++; + //let the index of value tag equal to alias + metadataValueTag[tag] = order[tag]; + + TableDesc valueTableDesc = conf.getValueTblDescs().get(tag); + SerDe valueSerDe = (SerDe) ReflectionUtils.newInstance(valueTableDesc + .getDeserializerClass(), null); + valueSerDe.initialize(null, valueTableDesc.getProperties()); + + MapJoinMetaData.put(Integer.valueOf(metadataValueTag[tag]), + new JDBMSinkObjectCtx(ObjectInspectorUtils + .getStandardObjectInspector(valueSerDe.getObjectInspector(), + ObjectInspectorCopyOption.WRITABLE), valueSerDe, + valueTableDesc, hconf)); + + + } + + // Construct externalizable objects for key and value + if (needNewKey) { + MapJoinObjectKey keyObj = new MapJoinObjectKey(metadataKeyTag, key); + MapJoinObjectValue valueObj = new MapJoinObjectValue( + metadataValueTag[tag], res); + + //valueObj.setConf(hconf); + valueObj.setConf(hconf); + // This may potentially increase the size of the hashmap on the mapper + if (res.size() > mapJoinRowsKey) { + if (res.size() % 100 == 0) { + LOG.warn("Number of values for a given key " + keyObj + " are " + + res.size()); + LOG.warn("used memory " + Runtime.getRuntime().totalMemory()); + } + } + hashTable.put(keyObj, valueObj); + } + }catch (SerDeException e) { + e.printStackTrace(); + throw new HiveException(e); + } + + } + + + @Override + /* + * Flush the hashtable into jdbm file + * Load this jdbm file into HDFS only + */ + public void closeOp(boolean abort) throws HiveException{ + try{ + if(mapJoinTables != null) { + //get tmp file URI + String tmpURI = this.getExecContext().getLocalWork().getTmpFileURI(); + LOG.info("Get TMP URI: "+tmpURI); + + for (Map.Entry> hashTables : mapJoinTables.entrySet()) { + //get the key and value + Byte tag = hashTables.getKey(); + HashMapWrapper hashTable = hashTables.getValue(); + + //get the jdbm file and path + String jdbmFile = hashTable.flushMemoryCacheToPersistent(); + Path localPath = new Path(jdbmFile); + + //get the tmp URI path; it will be a hdfs path if not local mode + Path tmpURIPath = new Path(tmpURI+Path.SEPARATOR+"-"+tag); + + //upload jdbm file to this HDFS + FileSystem fs = tmpURIPath.getFileSystem(this.getExecContext().getJc()); + fs.copyFromLocalFile(localPath, tmpURIPath); + LOG.info("Upload 1 JDBM File to: "+tmpURIPath); + //remove the original jdbm tmp file + hashTable.close(); + } + } + + super.closeOp(abort); + }catch(IOException e){ + LOG.error("Copy local file to HDFS error"); + e.printStackTrace(); + } + } + + /** + * Implements the getName function for the Node Interface. + * + * @return the name of the operator + */ + @Override + public String getName() { + return "JDBMSINK"; + } + + @Override + public int getType() { + return OperatorType.JDBMSINK; + } + + private void getPersistentFilePath(Map paths) throws HiveException{ + Map jdbmFilePaths = paths; + try{ + if(mapJoinTables != null) { + for (Map.Entry> hashTables : mapJoinTables.entrySet()) { + //hashTable.close(); + + Byte key = hashTables.getKey(); + HashMapWrapper hashTable = hashTables.getValue(); + + //get the jdbm file and path + String jdbmFile = hashTable.flushMemoryCacheToPersistent(); + Path localPath = new Path(jdbmFile); + + //insert into map + jdbmFilePaths.put(key, localPath); + } + } + }catch (Exception e){ + LOG.fatal("Get local JDBM file error"); + e.printStackTrace(); + } + } + + + /** + * Return the value as a standard object. StandardObject can be inspected by a + * standard ObjectInspector. + */ + protected static ArrayList computeValues(Object row, + List valueFields, List valueFieldsOI) + throws HiveException { + + // Compute the values + ArrayList nr = new ArrayList(valueFields.size()); + for (int i = 0; i < valueFields.size(); i++) { + if(valueFields.get(i) == null) { + continue; + } + + nr.add(ObjectInspectorUtils.copyToStandardObject(valueFields.get(i) + .evaluate(row), valueFieldsOI.get(i), + ObjectInspectorCopyOption.WRITABLE)); + } + + return nr; + } + + public TableDesc getSpillTableDesc(Byte alias) { + if (spillTableDesc == null || spillTableDesc.size() == 0) { + initSpillTables(); + } + return spillTableDesc.get(alias); + } + + public Map getSpillTableDesc() { + if (spillTableDesc == null) { + initSpillTables(); + } + return spillTableDesc; + } + + private SerDe getSpillSerDe(byte alias) { + TableDesc desc = getSpillTableDesc(alias); + if (desc == null) { + return null; + } + SerDe sd = (SerDe) ReflectionUtils.newInstance(desc.getDeserializerClass(), + null); + try { + sd.initialize(null, desc.getProperties()); + } catch (SerDeException e) { + e.printStackTrace(); + return null; + } + return sd; + } + + private void initSpillTables() { + Map> exprs = conf.getExprs(); + spillTableDesc = new HashMap(exprs.size()); + for (int tag = 0; tag < exprs.size(); tag++) { + List valueCols = exprs.get((byte) tag); + int columnSize = valueCols.size(); + StringBuilder colNames = new StringBuilder(); + StringBuilder colTypes = new StringBuilder(); + if (columnSize <= 0) { + continue; + } + for (int k = 0; k < columnSize; k++) { + String newColName = tag + "_VALUE_" + k; // any name, it does not + // matter. + colNames.append(newColName); + colNames.append(','); + colTypes.append(valueCols.get(k).getTypeString()); + colTypes.append(','); + } + // remove the last ',' + colNames.setLength(colNames.length() - 1); + colTypes.setLength(colTypes.length() - 1); + TableDesc tblDesc = new TableDesc(LazyBinarySerDe.class, + SequenceFileInputFormat.class, HiveSequenceFileOutputFormat.class, + Utilities.makeProperties( + org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "" + + Utilities.ctrlaCode, + org.apache.hadoop.hive.serde.Constants.LIST_COLUMNS, colNames + .toString(), + org.apache.hadoop.hive.serde.Constants.LIST_COLUMN_TYPES, + colTypes.toString())); + spillTableDesc.put((byte) tag, tblDesc); + } + } + + + private RowContainer getRowContainer(Configuration hconf, byte pos, Byte alias, + int containerSize) throws HiveException { + TableDesc tblDesc = getSpillTableDesc(alias); + SerDe serde = getSpillSerDe(alias); + + if (serde == null) { + containerSize = 1; + } + + RowContainer rc = new RowContainer(containerSize, hconf); + StructObjectInspector rcOI = null; + if (tblDesc != null) { + // arbitrary column names used internally for serializing to spill table + List colNames = Utilities.getColumnNames(tblDesc.getProperties()); + // object inspector for serializing input tuples + rcOI = ObjectInspectorFactory.getStandardStructObjectInspector(colNames, + joinValuesStandardObjectInspectors.get(pos)); + } + + rc.setSerDe(serde, rcOI); + rc.setTableDesc(tblDesc); + return rc; + } + + +} Property changes on: ql/src/java/org/apache/hadoop/hive/ql/exec/JDBMSinkOperator.java ___________________________________________________________________ Added: svn:executable + * Index: ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinMetaData.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinMetaData.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinMetaData.java (revision 0) @@ -0,0 +1,21 @@ +package org.apache.hadoop.hive.ql.exec; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.hive.ql.exec.JDBMSinkOperator.JDBMSinkObjectCtx; + +public class MapJoinMetaData { + static transient Map mapMetadata = new HashMap(); + + public MapJoinMetaData(){ + + } + public static void put(Integer key, JDBMSinkObjectCtx value){ + mapMetadata.put(key, value); + } + public static JDBMSinkObjectCtx get(Integer key){ + return mapMetadata.get(key); + } + +} Property changes on: ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinMetaData.java ___________________________________________________________________ Added: svn:executable + * Index: ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (revision 999637) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (working copy) @@ -17,8 +17,9 @@ */ package org.apache.hadoop.hive.ql.exec; - +import java.io.File; import java.io.Serializable; +import java.net.URI; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; @@ -26,18 +27,19 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.filecache.DistributedCache; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.exec.JDBMSinkOperator.JDBMSinkObjectCtx; import org.apache.hadoop.hive.ql.exec.persistence.HashMapWrapper; import org.apache.hadoop.hive.ql.exec.persistence.MapJoinObjectKey; import org.apache.hadoop.hive.ql.exec.persistence.MapJoinObjectValue; -import org.apache.hadoop.hive.ql.exec.persistence.RowContainer; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.MapJoinDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.plan.api.OperatorType; import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.SerDeException; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; import org.apache.hadoop.util.ReflectionUtils; @@ -51,6 +53,7 @@ private static final Log LOG = LogFactory.getLog(MapJoinOperator.class .getName()); + protected transient Map> mapJoinTables; private static final transient String[] FATAL_ERR_MSG = { @@ -59,63 +62,13 @@ + "Please increase that or remove the mapjoin hint." }; - /** - * MapJoinObjectCtx. - * - */ - public static class MapJoinObjectCtx { - ObjectInspector standardOI; - SerDe serde; - TableDesc tblDesc; - Configuration conf; - /** - * @param standardOI - * @param serde - */ - public MapJoinObjectCtx(ObjectInspector standardOI, SerDe serde, - TableDesc tblDesc, Configuration conf) { - this.standardOI = standardOI; - this.serde = serde; - this.tblDesc = tblDesc; - this.conf = conf; - } - - /** - * @return the standardOI - */ - public ObjectInspector getStandardOI() { - return standardOI; - } - /** - * @return the serde - */ - public SerDe getSerDe() { - return serde; - } - - public TableDesc getTblDesc() { - return tblDesc; - } - - public Configuration getConf() { - return conf; - } - - } - - static transient Map mapMetadata = new HashMap(); - static transient int nextVal = 0; - - public static Map getMapMetadata() { - return mapMetadata; - } transient int metadataKeyTag; transient int[] metadataValueTag; transient int maxMapJoinSize; - + private int bigTableAlias; public MapJoinOperator() { } @@ -125,6 +78,7 @@ @Override protected void initializeOp(Configuration hconf) throws HiveException { + super.initializeOp(hconf); maxMapJoinSize = HiveConf.getIntVar(hconf, @@ -135,6 +89,9 @@ metadataValueTag[pos] = -1; } + metadataKeyTag = -1; + bigTableAlias = order[posBigTable]; + mapJoinTables = new HashMap>(); // initialize the hash tables for other tables @@ -150,138 +107,159 @@ mapJoinTables.put(Byte.valueOf((byte) pos), hashTable); } + } + @Override protected void fatalErrorMessage(StringBuilder errMsg, long counterCode) { errMsg.append("Operator " + getOperatorId() + " (id=" + id + "): " + FATAL_ERR_MSG[(int) counterCode]); } - @Override - public void processOp(Object row, int tag) throws HiveException { + public ArrayList computeKey(Object row, Byte alias) throws HiveException{ + return computeValues(row, joinKeys.get(alias),joinKeysObjectInspectors.get(alias)); + } + + public ArrayList computeValue(Object row, Byte alias) throws HiveException{ + return computeValues(row, joinValues.get(alias),joinValuesObjectInspectors.get(alias)); + } - if (tag == posBigTable) { - this.getExecContext().processInputFileChangeForLocalWork(); - } + public void generateMapMetaData() throws HiveException,SerDeException{ + //generate the meta data for key + //index for key is -1 + TableDesc keyTableDesc = conf.getKeyTblDesc(); + SerDe keySerializer = (SerDe) ReflectionUtils.newInstance( + keyTableDesc.getDeserializerClass(), null); + keySerializer.initialize(null, keyTableDesc.getProperties()); + MapJoinMetaData.put(Integer.valueOf(metadataKeyTag), + new JDBMSinkObjectCtx( + ObjectInspectorUtils + .getStandardObjectInspector(keySerializer + .getObjectInspector(), + ObjectInspectorCopyOption.WRITABLE), keySerializer, + keyTableDesc, hconf)); - try { - // get alias - alias = (byte) tag; + //index for values is just alias + for (int tag = 0; tag < order.length; tag++) { + int alias = (int) order[tag]; - if ((lastAlias == null) || (!lastAlias.equals(alias))) { - nextSz = joinEmitInterval; + if(alias == this.bigTableAlias){ + continue; } - // compute keys and values as StandardObjects - ArrayList key = computeKeys(row, joinKeys.get(alias), - joinKeysObjectInspectors.get(alias)); - ArrayList value = computeValues(row, joinValues.get(alias), - joinValuesObjectInspectors.get(alias), joinFilters.get(alias), - joinFilterObjectInspectors.get(alias), noOuterJoin); - // does this source need to be stored in the hash map - if (tag != posBigTable) { - if (firstRow) { - metadataKeyTag = nextVal++; + TableDesc valueTableDesc = conf.getValueTblDescs().get(tag); + SerDe valueSerDe = (SerDe) ReflectionUtils.newInstance(valueTableDesc + .getDeserializerClass(), null); + valueSerDe.initialize(null, valueTableDesc.getProperties()); - TableDesc keyTableDesc = conf.getKeyTblDesc(); - SerDe keySerializer = (SerDe) ReflectionUtils.newInstance( - keyTableDesc.getDeserializerClass(), null); - keySerializer.initialize(null, keyTableDesc.getProperties()); + MapJoinMetaData.put(Integer.valueOf(alias), + new JDBMSinkObjectCtx(ObjectInspectorUtils + .getStandardObjectInspector(valueSerDe.getObjectInspector(), + ObjectInspectorCopyOption.WRITABLE), valueSerDe, + valueTableDesc, hconf)); + } + } - mapMetadata.put(Integer.valueOf(metadataKeyTag), - new MapJoinObjectCtx( - ObjectInspectorUtils - .getStandardObjectInspector(keySerializer - .getObjectInspector(), - ObjectInspectorCopyOption.WRITABLE), keySerializer, - keyTableDesc, hconf)); + private void loadJDBM() throws HiveException{ + boolean localMode = HiveConf.getVar(hconf, HiveConf.ConfVars.HADOOPJT).equals("local"); + String tmpURI =null; + HashMapWrapper hashtable; + Byte pos; + int alias; + try{ + if(localMode){ + //load the jdbm file from tmp dir + tmpURI= this.getExecContext().getLocalWork().getTmpFileURI(); + for(Map.Entry> entry: mapJoinTables.entrySet()){ + pos = entry.getKey(); + hashtable=entry.getValue(); + URI uri = new URI(tmpURI+Path.SEPARATOR+"-"+pos); - firstRow = false; + File jdbmFile = new File(uri); + hashtable.initilizePersistentHash(jdbmFile); } + }else{ + //load the jdbm file from distributed cache + LOG.info("******* Load from distributed Cache ***:"); + Path[] localFiles= DistributedCache.getLocalCacheFiles(this.hconf); + for(int i = 0;i> entry: mapJoinTables.entrySet()){ + pos = entry.getKey(); + hashtable=entry.getValue(); + String suffix="-"+pos; + LOG.info("Looking for jdbm file with suffix: "+suffix); - reportProgress(); - numMapRowsRead++; + boolean found=false; + for(int i = 0;i maxMapJoinSize) && (reporter != null) - && (counterNameToEnum != null)) { - // update counter - LOG - .warn("Too many rows in map join tables. Fatal error counter will be incremented!!"); - incrCounter(fatalErrorCntr, 1); - fatalError = true; - return; - } + if(path.toString().endsWith(suffix)){ + LOG.info("Matching suffix with cached file:"+path.toString()); + File jdbmFile = new File(path.toString()); + LOG.info("\tInitializing the JDBM by cached file:"+path.toString()); + hashtable.initilizePersistentHash(jdbmFile); + found = true; + LOG.info("\tLoad back 1 JDBM file from distributed cache:"+path.toString()); + break; + } + } + if(!found){ + LOG.error("Load nothing from Distributed Cache"); + throw new HiveException(); + } + } + LOG.info("******* End of loading *******:"); + + } + }catch (Exception e){ + LOG.error("Load Hash Table error"); - HashMapWrapper hashTable = mapJoinTables - .get(alias); - MapJoinObjectKey keyMap = new MapJoinObjectKey(metadataKeyTag, key); - MapJoinObjectValue o = hashTable.get(keyMap); - RowContainer res = null; + throw new HiveException(); + } - boolean needNewKey = true; - if (o == null) { - int bucketSize = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEMAPJOINBUCKETCACHESIZE); - res = getRowContainer(hconf, (byte) tag, order[tag], bucketSize); - res.add(value); - } else { - res = o.getObj(); - res.add(value); - // If key already exists, HashMapWrapper.get() guarantees it is - // already in main memory HashMap - // cache. So just replacing the object value should update the - // HashMapWrapper. This will save - // the cost of constructing the new key/object and deleting old one - // and inserting the new one. - if (hashTable.cacheSize() > 0) { - o.setObj(res); - needNewKey = false; - } - } - if (metadataValueTag[tag] == -1) { - metadataValueTag[tag] = nextVal++; + } - TableDesc valueTableDesc = conf.getValueTblDescs().get(tag); - SerDe valueSerDe = (SerDe) ReflectionUtils.newInstance(valueTableDesc - .getDeserializerClass(), null); - valueSerDe.initialize(null, valueTableDesc.getProperties()); + @Override + public void processOp(Object row, int tag) throws HiveException { - mapMetadata.put(Integer.valueOf(metadataValueTag[tag]), - new MapJoinObjectCtx(ObjectInspectorUtils - .getStandardObjectInspector(valueSerDe.getObjectInspector(), - ObjectInspectorCopyOption.WRITABLE), valueSerDe, - valueTableDesc, hconf)); - } + try { + if(firstRow){ + //generate the map metadata + generateMapMetaData(); + //load the distributed cache jdbm file into hashtable + loadJDBM(); + firstRow = false; + } + + // get alias + alias = order[tag]; + //alias = (byte)tag; - // Construct externalizable objects for key and value - if (needNewKey) { - MapJoinObjectKey keyObj = new MapJoinObjectKey(metadataKeyTag, key); - MapJoinObjectValue valueObj = new MapJoinObjectValue( - metadataValueTag[tag], res); - valueObj.setConf(hconf); - valueObj.setConf(hconf); - // This may potentially increase the size of the hashmap on the mapper - if (res.size() > mapJoinRowsKey) { - if (res.size() % 100 == 0) { - LOG.warn("Number of values for a given key " + keyObj + " are " - + res.size()); - LOG.warn("used memory " + Runtime.getRuntime().totalMemory()); - } - } - hashTable.put(keyObj, valueObj); - } - return; + if ((lastAlias == null) || (!lastAlias.equals(alias))) { + nextSz = joinEmitInterval; } + // compute keys and values as StandardObjects + ArrayList key = computeValues(row, joinKeys.get(alias), + joinKeysObjectInspectors.get(alias)); + ArrayList value = computeValues(row, joinValues.get(alias), + joinValuesObjectInspectors.get(alias)); + + // Add the value to the ArrayList - storage.get(alias).add(value); + storage.get((byte)tag).add(value); for (Byte pos : order) { if (pos.intValue() != tag) { MapJoinObjectKey keyMap = new MapJoinObjectKey(metadataKeyTag, key); - MapJoinObjectValue o = mapJoinTables.get(pos).get(keyMap); + MapJoinObjectValue o = mapJoinTables.get(pos).getMapJoinValueObject(keyMap); // there is no join-value or join-key has all null elements if (o == null || (hasAnyNulls(key))) { @@ -300,7 +278,7 @@ checkAndGenObject(); // done with the row - storage.get(alias).clear(); + storage.get((byte)tag).clear(); for (Byte pos : order) { if (pos.intValue() != tag) { @@ -316,6 +294,7 @@ @Override public void closeOp(boolean abort) throws HiveException { + if(mapJoinTables != null) { for (HashMapWrapper hashTable : mapJoinTables.values()) { hashTable.close(); Index: ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java (revision 997835) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java (working copy) @@ -18,28 +18,26 @@ package org.apache.hadoop.hive.ql.exec; +import java.io.IOException; import java.io.OutputStream; import java.io.Serializable; -import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.Properties; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.fs.ContentSummary; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.ql.QueryPlan; +import org.apache.hadoop.hive.ql.Context; +import org.apache.hadoop.hive.ql.DriverContext; import org.apache.hadoop.hive.ql.exec.Utilities.StreamPrinter; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.MapredWork; -import org.apache.hadoop.hive.ql.plan.api.StageType; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.shims.ShimLoader; -import org.apache.hadoop.hive.ql.DriverContext; -import org.apache.hadoop.hive.ql.Context; -import org.apache.hadoop.fs.ContentSummary; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.mapred.JobConf; /** @@ -87,14 +85,15 @@ if (!ctx.isLocalOnlyExecutionMode() && conf.getBoolVar(HiveConf.ConfVars.LOCALMODEAUTO)) { - if (inputSummary == null) + if (inputSummary == null) { inputSummary = Utilities.getInputSummary(driverContext.getCtx(), work, null); + } // at this point the number of reducers is precisely defined in the plan int numReducers = work.getNumReduceTasks(); if (LOG.isDebugEnabled()) { - LOG.debug("Task: " + getId() + ", Summary: " + + LOG.debug("Task: " + getId() + ", Summary: " + inputSummary.getLength() + "," + inputSummary.getFileCount() + "," + numReducers); } @@ -117,7 +116,9 @@ if(!runningViaChild) { // we are not running this mapred task via child jvm // so directly invoke ExecDriver - return super.execute(driverContext); + int exitcode = super.execute(driverContext); + return exitcode; + } // enable assertion @@ -239,8 +240,9 @@ // creating the context can create a bunch of files. So make // sure to clear it out - if(ctxCreated) + if(ctxCreated) { ctx.clear(); + } } catch (Exception e) { LOG.error("Exception: " + e.getMessage()); @@ -324,9 +326,10 @@ long bytesPerReducer = conf.getLongVar(HiveConf.ConfVars.BYTESPERREDUCER); int maxReducers = conf.getIntVar(HiveConf.ConfVars.MAXREDUCERS); - if(inputSummary == null) + if(inputSummary == null) { // compute the summary and stash it away inputSummary = Utilities.getInputSummary(driverContext.getCtx(), work, null); + } long totalInputFileSize = inputSummary.getLength(); @@ -355,23 +358,26 @@ long maxTasks = conf.getIntVar(HiveConf.ConfVars.LOCALMODEMAXTASKS); // check for max input size - if (inputSummary.getLength() > maxBytes) + if (inputSummary.getLength() > maxBytes) { return "Input Size (= " + inputSummary.getLength() + ") is larger than " + HiveConf.ConfVars.LOCALMODEMAXBYTES.varname + " (= " + maxBytes + ")"; + } // ideally we would like to do this check based on the number of splits // in the absence of an easy way to get the number of splits - do this // based on the total number of files (pessimistically assumming that // splits are equal to number of files in worst case) - if (inputSummary.getFileCount() > maxTasks) + if (inputSummary.getFileCount() > maxTasks) { return "Number of Input Files (= " + inputSummary.getFileCount() + - ") is larger than " + + ") is larger than " + HiveConf.ConfVars.LOCALMODEMAXTASKS.varname + "(= " + maxTasks + ")"; + } // since local mode only runs with 1 reducers - make sure that the // the number of reducers (set by user or inferred) is <=1 - if (numReducers > 1) + if (numReducers > 1) { return "Number of reducers (= " + numReducers + ") is more than 1"; + } return null; } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java (revision 0) @@ -0,0 +1,186 @@ +package org.apache.hadoop.hive.ql.exec; + +import java.io.File; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.Context; +import org.apache.hadoop.hive.ql.DriverContext; +import org.apache.hadoop.hive.ql.QueryPlan; +import org.apache.hadoop.hive.ql.plan.FetchWork; +import org.apache.hadoop.hive.ql.plan.MapredLocalWork; +import org.apache.hadoop.hive.ql.plan.api.StageType; +import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; +import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.mapred.JobConf; +public class MapredLocalTask extends Task implements Serializable { + + private Map fetchOperators; + private File jdbmFile; + private JobConf job; + public static final Log l4j = LogFactory.getLog("MapredLocalTask"); + private MapOperator mo; + // not sure we need this exec context; but all the operators in the work + // will pass this context throught + private final ExecMapperContext execContext = new ExecMapperContext(); + + public MapredLocalTask(){ + super(); + } + + @Override + public void initialize(HiveConf conf, QueryPlan queryPlan, + DriverContext driverContext) { + super.initialize(conf, queryPlan, driverContext); + job = new JobConf(conf, ExecDriver.class); + } + + @Override +public int execute(DriverContext driverContext){ + // check the local work + if(work == null){ + return -1; + } + + //to do: to handle multiple small tables + processMapJoinLocalWork(driverContext); + + + return 0; + } + + + /** + * Process the Map Join local work + * (Let's assume no input file change first) + * + */ + private void processMapJoinLocalWork(DriverContext driverContext){ + + fetchOperators = new HashMap(); + Map fetchOpJobConfMap = new HashMap(); + + execContext.setJc(job); + //set the local work, so all the operator can get this context + execContext.setLocalWork(work); + + + try{ + // this mapper operator is used to initialize all the operators + + for (Map.Entry entry : work.getAliasToFetchWork().entrySet()) { + JobConf jobClone = new JobConf(job); + + Operator tableScan = work.getAliasToWork().get(entry.getKey()); + boolean setColumnsNeeded = false; + if(tableScan instanceof TableScanOperator) { + ArrayList list = ((TableScanOperator)tableScan).getNeededColumnIDs(); + if (list != null) { + ColumnProjectionUtils.appendReadColumnIDs(jobClone, list); + setColumnsNeeded = true; + } + } + + if (!setColumnsNeeded) { + ColumnProjectionUtils.setFullyReadColumns(jobClone); + } + + //create a fetch operator + FetchOperator fetchOp = new FetchOperator(entry.getValue(),jobClone); + fetchOpJobConfMap.put(fetchOp, jobClone); + fetchOperators.put(entry.getKey(), fetchOp); + l4j.info("fetchoperator for " + entry.getKey() + " created"); + } + + //initilize all forward operator + for (Map.Entry entry : fetchOperators.entrySet()) { + //get the forward op + Operator forwardOp = work.getAliasToWork().get(entry.getKey()); + + //put the exe context into all the operators + forwardOp.setExecContext(execContext); + // All the operators need to be initialized before process + FetchOperator fetchOp = entry.getValue(); + JobConf jobConf = fetchOpJobConfMap.get(fetchOp); + + if (jobConf == null) { + jobConf = job; + } + + + //initialize the forward operator + forwardOp.initialize(jobConf, new ObjectInspector[] {fetchOp.getOutputObjectInspector()}); + l4j.info("fetchoperator for " + entry.getKey() + " initialized"); + } + + + //start to execute the local work + int fetchOpNum = 0; + for (Map.Entry entry : fetchOperators.entrySet()) { + int fetchOpRows = 0; + String alias = entry.getKey(); + FetchOperator fetchOp = entry.getValue(); + + /* skip these code right now + if (inputFileChangeSenstive) { + fetchOp.clearFetchContext(); + setUpFetchOpContext(fetchOp, alias); + }*/ + + //get the root operator + Operator forwardOp = work.getAliasToWork().get(alias); + //walk through the operator tree + while (true) { + InspectableObject row = fetchOp.getNextRow(); + if (row == null) { + forwardOp.close(false); + break; + } + fetchOpRows++; + forwardOp.process(row.o, 0); + // check if any operator had a fatal error or early exit during + // execution + if (forwardOp.getDone()) { + //ExecMapper.setDone(true); + break; + } + } + } + } catch (Throwable e) { + if (e instanceof OutOfMemoryError) { + // Don't create a new object if we are already out of memory + l4j.error("Out of Merror Error"); + } else { + l4j.error("Hive Runtime Error: Map local work failed"); + e.printStackTrace(); + } + } + + + } + + + + @Override + public void localizeMRTmpFilesImpl(Context ctx){ + + + } + + @Override + public String getName() { + return "MAPREDLOCAL"; + } + @Override + public int getType() { + //assert false; + return StageType.MAPREDLOCAL; + } + +} Property changes on: ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java ___________________________________________________________________ Added: svn:executable + * Index: ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (revision 997835) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (working copy) @@ -197,6 +197,8 @@ // object inspectors for input rows // We will increase the size of the array on demand protected transient ObjectInspector[] inputObjInspectors = new ObjectInspector[1]; + + // for output rows of this operator protected transient ObjectInspector outputObjInspector; @@ -281,6 +283,9 @@ return true; } for (Operator parent : parentOperators) { + if (parent == null) { + return true; + } if (parent.state != State.INIT) { return false; } @@ -319,6 +324,7 @@ // while // initializing so this need to be done here instead of initialize() method if (childOperators != null) { + childOperatorsArray = new Operator[childOperators.size()]; for (int i = 0; i < childOperatorsArray.length; i++) { childOperatorsArray[i] = childOperators.get(i); @@ -427,6 +433,14 @@ initialize(hconf, null); } + public ObjectInspector[] getInputObjInspectors() { + return inputObjInspectors; + } + + public void setInputObjInspectors(ObjectInspector[] inputObjInspectors) { + this.inputObjInspectors = inputObjInspectors; + } + /** * Process the row. * @@ -501,6 +515,9 @@ protected boolean allInitializedParentsAreClosed() { if (parentOperators != null) { for (Operator parent : parentOperators) { + if(parent==null){ + continue; + } if (!(parent.state == State.CLOSE || parent.state == State.UNINIT)) { return false; } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java (revision 997835) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java (working copy) @@ -28,6 +28,7 @@ import org.apache.hadoop.hive.ql.plan.FilterDesc; import org.apache.hadoop.hive.ql.plan.ForwardDesc; import org.apache.hadoop.hive.ql.plan.GroupByDesc; +import org.apache.hadoop.hive.ql.plan.JDBMDummyDesc; import org.apache.hadoop.hive.ql.plan.JoinDesc; import org.apache.hadoop.hive.ql.plan.LateralViewForwardDesc; import org.apache.hadoop.hive.ql.plan.LateralViewJoinDesc; @@ -85,6 +86,8 @@ LateralViewJoinOperator.class)); opvec.add(new OpTuple(LateralViewForwardDesc.class, LateralViewForwardOperator.class)); + opvec.add(new OpTuple(JDBMDummyDesc.class, + JDBMDummyOperator.class)); } public static Operator get(Class opClass) { Index: ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java (revision 997835) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java (working copy) @@ -29,6 +29,7 @@ import org.apache.hadoop.hive.ql.plan.ExplainWork; import org.apache.hadoop.hive.ql.plan.FetchWork; import org.apache.hadoop.hive.ql.plan.FunctionWork; +import org.apache.hadoop.hive.ql.plan.MapredLocalWork; import org.apache.hadoop.hive.ql.plan.MapredWork; import org.apache.hadoop.hive.ql.plan.MoveWork; @@ -67,6 +68,9 @@ ConditionalTask.class)); taskvec.add(new taskTuple(MapredWork.class, MapRedTask.class)); + + taskvec.add(new taskTuple(MapredLocalWork.class, + MapredLocalTask.class)); } private static ThreadLocal tid = new ThreadLocal() { @@ -130,7 +134,7 @@ } makeChild(ret, tasklist); - + return (ret); } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (revision 997835) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (working copy) @@ -85,8 +85,8 @@ import org.apache.hadoop.hive.ql.plan.MapredWork; import org.apache.hadoop.hive.ql.plan.PartitionDesc; import org.apache.hadoop.hive.ql.plan.PlanUtils; +import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.plan.PlanUtils.ExpressionTypes; -import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.apache.hadoop.hive.shims.ShimLoader; @@ -396,10 +396,14 @@ public static void serializeMapRedWork(MapredWork w, OutputStream out) { XMLEncoder e = new XMLEncoder(out); // workaround for java 1.5 - e.setPersistenceDelegate(ExpressionTypes.class, new EnumDelegate()); - e.setPersistenceDelegate(GroupByDesc.Mode.class, new EnumDelegate()); - e.writeObject(w); - e.close(); + try{ + e.setPersistenceDelegate(ExpressionTypes.class, new EnumDelegate()); + e.setPersistenceDelegate(GroupByDesc.Mode.class, new EnumDelegate()); + e.writeObject(w); + e.close(); + }catch(Exception ex){ + ex.printStackTrace(); + } } public static MapredWork deserializeMapRedWork(InputStream in, Index: ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java (revision 997835) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java (working copy) @@ -21,6 +21,7 @@ import java.io.File; import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.Properties; import java.util.Set; @@ -72,7 +73,7 @@ /** * Constructor. - * + * * @param threshold * User specified threshold to store new values into persistent * storage. @@ -91,16 +92,73 @@ } /** - * Get the value based on the key. We try to get it from the main memory hash - * table first. If it is not there we will look up the persistent hash table. - * This function also guarantees if any item is found given a key, it is - * available in main memory HashMap. So mutating the returned value will be - * reflected (saved) in HashMapWrapper. - * + * Get the value based on the key. this GET method will directly + * return the value from jdbm storage. * @param key * @return Value corresponding to the key. If the key is not found, return * null. */ +/* + public V getMapJoinValueObject(K key) throws HiveException{ + if(pHash == null) { + LOG.warn("the jdbm object is not ready!"); + throw new HiveException(); + } + try{ + V value = (V)pHash.get(key); + return value; + }catch(Exception e){ + throw new HiveException(e); + } + }*/ + + /* + * In this get operation, the jdbm should read only + */ + public V getMapJoinValueObject(K key) throws HiveException { + V value = null; + + // if not the MRU, searching the main memory hash table. + MRUItem item = mHash.get(key); + if (item != null) { + value = item.value; + MRUList.moveToHead(item); + } else if (pHash != null) { + try { + value = (V) pHash.get(key); + if (value != null) { + if (mHash.size() < threshold) { + MRUItem itm= new MRUItem(key, value); + mHash.put(key, itm); + //pHash.remove(key); + MRUList.put(itm); + //recman.commit(); + + } else if (threshold > 0) { // flush the LRU to disk + MRUItem tail = MRUList.tail(); // least recently used item + //pHash.put(tail.key, tail.value); + //pHash.remove(key); + //recman.commit(); + + // update mHash -- reuse MRUItem + item = mHash.remove(tail.key); + item.key = key; + item.value = value; + mHash.put(key, item); + + // update MRU -- reusing MRUItem + tail.key = key; + tail.value = value; + MRUList.moveToHead(tail); + } + } + } catch (Exception e) { + LOG.warn(e.toString()); + throw new HiveException(e); + } + } + return value; + } public V get(K key) throws HiveException { V value = null; @@ -146,7 +204,7 @@ * Put the key value pair in the hash table. It will first try to put it into * the main memory hash table. If the size exceeds the threshold, it will put * it into the persistent hash table. - * + * * @param key * @param value * @throws HiveException @@ -208,9 +266,82 @@ } } + public void putToJDBM(K key, V value) throws HiveException{ + if (pHash == null) { + pHash = getPersistentHash(); + } + try { + pHash.put(key, value); + recman.commit(); + } catch (Exception e) { + LOG.warn(e.toString()); + throw new HiveException(e); + } + + } + + /** + * Flush the main memory hash table into the persistent cache file + * + * @return persistent cache file + */ + public String flushMemoryCacheToPersistent() throws HiveException{ + int mm_size = mHash.size(); + //no data in the memory cache + if(mm_size == 0){ + return null; + } + try{ + //if no persistent cache file; create a new one + if(pHash == null){ + pHash = getPersistentHash(); + } + + //iterate the memory hash table and put them into persistent file + for (Map.Entry entry : mHash.entrySet()) { + K key = entry.getKey(); + MRUItem item = entry.getValue(); + pHash.put(key, item.value); + } + //commit to the persistent file + recman.commit(); + + //release the memory + mHash.clear(); + + }catch (Exception e) { + LOG.warn(e.toString()); + throw new HiveException(e); + } + return tmpFile.getAbsolutePath(); + } + + public void initilizePersistentHash(File jdbmfile) throws HiveException{ + try{ + Properties props = new Properties(); + props.setProperty(RecordManagerOptions.CACHE_TYPE, + RecordManagerOptions.NO_CACHE); + props.setProperty(RecordManagerOptions.DISABLE_TRANSACTIONS, "true"); + + recman = RecordManagerFactory.createRecordManager(jdbmfile, props); + long recid = recman.getNamedObject( "hashtable" ); + if ( recid != 0 ) { + System.out.println( "Reloading existing hashtable..." ); + pHash = HTree.load( recman, recid ); + }else{ + LOG.warn("initiliaze the hash table by jdbm file Error!"); + throw new HiveException(); + } + + } catch (Exception e) { + LOG.warn(e.toString()); + throw new HiveException(e); + } + } + /** * Get the persistent hash table. - * + * * @return persistent hash table * @throws HiveException */ @@ -234,6 +365,7 @@ recman = RecordManagerFactory.createRecordManager(tmpFile, props); pHash = HTree.createInstance(recman); + recman.setNamedObject( "hashtable", pHash.getRecid() ); } catch (Exception e) { LOG.warn(e.toString()); throw new HiveException(e); @@ -259,7 +391,7 @@ * the pairs are removed from the main memory hash table, pairs in the * persistent hash table will not be moved to the main memory hash table. * Future inserted elements will go into the main memory hash table though. - * + * * @param key * @throws HiveException */ @@ -279,7 +411,7 @@ /** * Get a list of all keys in the hash map. - * + * * @return */ public Set keySet() { @@ -306,7 +438,7 @@ /** * Get the main memory cache capacity. - * + * * @return the maximum number of items can be put into main memory HashMap * cache. */ @@ -316,7 +448,7 @@ /** * Close the persistent hash table and clean it up. - * + * * @throws HiveException */ public void close() throws HiveException { @@ -330,8 +462,10 @@ throw new HiveException(e); } // delete the temporary file - tmpFile.delete(); - tmpFile = null; + if(tmpFile != null){ + tmpFile.delete(); + tmpFile = null; + } pHash = null; recman = null; } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectKey.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectKey.java (revision 997835) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectKey.java (working copy) @@ -24,8 +24,8 @@ import java.io.ObjectOutput; import java.util.ArrayList; -import org.apache.hadoop.hive.ql.exec.MapJoinOperator; -import org.apache.hadoop.hive.ql.exec.MapJoinOperator.MapJoinObjectCtx; +import org.apache.hadoop.hive.ql.exec.MapJoinMetaData; +import org.apache.hadoop.hive.ql.exec.JDBMSinkOperator.JDBMSinkObjectCtx; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; @@ -81,7 +81,7 @@ metadataTag = in.readInt(); // get the tableDesc from the map stored in the mapjoin operator - MapJoinObjectCtx ctx = MapJoinOperator.getMapMetadata().get( + JDBMSinkObjectCtx ctx = MapJoinMetaData.get( Integer.valueOf(metadataTag)); Writable val = ctx.getSerDe().getSerializedClass().newInstance(); @@ -99,9 +99,8 @@ public void writeExternal(ObjectOutput out) throws IOException { try { out.writeInt(metadataTag); - // get the tableDesc from the map stored in the mapjoin operator - MapJoinObjectCtx ctx = MapJoinOperator.getMapMetadata().get( + JDBMSinkObjectCtx ctx = MapJoinMetaData.get( Integer.valueOf(metadataTag)); // Different processing for key and value Index: ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectValue.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectValue.java (revision 997835) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectValue.java (working copy) @@ -26,8 +26,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.exec.MapJoinOperator; -import org.apache.hadoop.hive.ql.exec.MapJoinOperator.MapJoinObjectCtx; +import org.apache.hadoop.hive.ql.exec.MapJoinMetaData; +import org.apache.hadoop.hive.ql.exec.JDBMSinkOperator.JDBMSinkObjectCtx; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; @@ -84,10 +84,11 @@ public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { try { + metadataTag = in.readInt(); // get the tableDesc from the map stored in the mapjoin operator - MapJoinObjectCtx ctx = MapJoinOperator.getMapMetadata().get( + JDBMSinkObjectCtx ctx = MapJoinMetaData.get( Integer.valueOf(metadataTag)); int sz = in.readInt(); RowContainer res = new RowContainer(bucketSize, ctx.getConf()); @@ -113,11 +114,10 @@ @Override public void writeExternal(ObjectOutput out) throws IOException { try { - out.writeInt(metadataTag); // get the tableDesc from the map stored in the mapjoin operator - MapJoinObjectCtx ctx = MapJoinOperator.getMapMetadata().get( + JDBMSinkObjectCtx ctx = MapJoinMetaData.get( Integer.valueOf(metadataTag)); // Different processing for key and value Index: ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java (revision 997835) +++ ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java (working copy) @@ -29,12 +29,11 @@ */ public class IOContext { - private static ThreadLocal threadLocal = new ThreadLocal(); - static { - if (threadLocal.get() == null) { - threadLocal.set(new IOContext()); - } - } + + private static ThreadLocal threadLocal = new ThreadLocal(){ + @Override + protected synchronized IOContext initialValue() { return new IOContext(); } + }; public static IOContext get() { return IOContext.threadLocal.get(); Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java (revision 0) @@ -0,0 +1,162 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.optimizer.physical; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Stack; + +import org.apache.hadoop.hive.ql.exec.JDBMDummyOperator; +import org.apache.hadoop.hive.ql.exec.JDBMSinkOperator; +import org.apache.hadoop.hive.ql.exec.MapJoinOperator; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.OperatorFactory; +import org.apache.hadoop.hive.ql.exec.RowSchema; +import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.lib.NodeProcessor; +import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; +import org.apache.hadoop.hive.ql.optimizer.physical.MapJoinResolver.LocalMapJoinProcCtx; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.JDBMDummyDesc; +import org.apache.hadoop.hive.ql.plan.PlanUtils; +import org.apache.hadoop.hive.ql.plan.TableDesc; + +/** + * Node processor factory for skew join resolver. + */ +public final class LocalMapJoinProcFactory { + + + + public static NodeProcessor getJoinProc() { + return new LocalMapJoinProcessor(); + } + public static NodeProcessor getMapJoinMapJoinProc() { + return new MapJoinMapJoinProc(); + } + public static NodeProcessor getDefaultProc() { + return new NodeProcessor() { + @Override + public Object process(Node nd, Stack stack, + NodeProcessorCtx procCtx, Object... nodeOutputs) + throws SemanticException { + return null; + } + }; + } + + /** + * LocalMapJoinProcessor. + * + */ + public static class LocalMapJoinProcessor implements NodeProcessor { + public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, + Object... nodeOutputs) throws SemanticException { + LocalMapJoinProcCtx context = (LocalMapJoinProcCtx) ctx; + + if(!nd.getName().equals("MAPJOIN")){ + return null; + } + MapJoinOperator mapJoinOp = (MapJoinOperator) nd; + JDBMSinkOperator jdbmSinkOp = new JDBMSinkOperator(mapJoinOp); + + //get the last operator for processing big tables + int bigTable = mapJoinOp.getConf().getPosBigTable(); + Byte[] order = mapJoinOp.getConf().getTagOrder(); + int bigTableAlias=(int)order[bigTable]; + + Operator bigOp = mapJoinOp.getParentOperators().get(bigTable); + + //the parent ops for jdbmSinkOp + List> smallTablesParentOp= new ArrayList>(); + + List> dummyOperators= new ArrayList>(); + //get all parents + List > parentsOp = mapJoinOp.getParentOperators(); + for(int i = 0; i parent = parentsOp.get(i); + //let jdbmOp be the child of this parent + parent.replaceChild(mapJoinOp, jdbmSinkOp); + //keep the parent id correct + smallTablesParentOp.add(parent); + + //get parent schema + RowSchema rowSchema = parent.getSchema(); + + //create an new operator: JDBMDummyOpeator, which share the table desc + JDBMDummyDesc desc = new JDBMDummyDesc(); + TableDesc tbl = PlanUtils.getIntermediateFileTableDesc(PlanUtils + .getFieldSchemasFromRowSchema(rowSchema, "")); + desc.setTbl(tbl); + + JDBMDummyOperator dummyOp =(JDBMDummyOperator)OperatorFactory.get(desc); + + dummyOp.getConf().setTbl(tbl); + + + //let the dummy op be the parent of mapjoin op + mapJoinOp.replaceParent(parent, dummyOp); + List> dummyChildren = new ArrayList>(); + dummyChildren.add(mapJoinOp); + dummyOp.setChildOperators(dummyChildren); + + //add this dummy op to the dummp operator list + dummyOperators.add(dummyOp); + + } + + jdbmSinkOp.setParentOperators(smallTablesParentOp); + for(Operator op: dummyOperators){ + context.addDummyParentOp(op); + } + + return null; + } + + } + + /** + * LocalMapJoinProcessor. + * + */ + public static class MapJoinMapJoinProc implements NodeProcessor { + public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, + Object... nodeOutputs) throws SemanticException { + LocalMapJoinProcCtx context = (LocalMapJoinProcCtx) ctx; + if(!nd.getName().equals("MAPJOIN")){ + return null; + } + System.out.println("Mapjoin * MapJoin"); + + return null; + } + } + + + private LocalMapJoinProcFactory() { + // prevent instantiation + } + } + Property changes on: ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java ___________________________________________________________________ Added: svn:executable + * Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java (revision 0) @@ -0,0 +1,230 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.optimizer.physical; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Stack; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.Context; +import org.apache.hadoop.hive.ql.exec.ConditionalTask; +import org.apache.hadoop.hive.ql.exec.MapredLocalTask; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; +import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; +import org.apache.hadoop.hive.ql.lib.Dispatcher; +import org.apache.hadoop.hive.ql.lib.GraphWalker; +import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.lib.NodeProcessor; +import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; +import org.apache.hadoop.hive.ql.lib.Rule; +import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.parse.ParseContext; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.MapredLocalWork; +import org.apache.hadoop.hive.ql.plan.MapredWork; + +/** + * An implementation of PhysicalPlanResolver. It iterator each MapRedTask to see whether the task has a local map work + * if it has, it will move the local work to a new local map join task. Then it will make this new generated task depends on + * current task's parent task and make current task depends on this new generated task. + */ +public class MapJoinResolver implements PhysicalPlanResolver { + @Override + public PhysicalContext resolve(PhysicalContext pctx) throws SemanticException { + //create dispatcher and graph walker + Dispatcher disp = new LocalMapJoinTaskDispatcher(pctx); + GraphWalker ogw = new DefaultGraphWalker(disp); + + //get all the tasks nodes from root task + ArrayList topNodes = new ArrayList(); + topNodes.addAll(pctx.rootTasks); + + //begin to walk through the task tree. + ogw.startWalking(topNodes, null); + return null; + } + + /** + * Iterator each tasks. If this task has a local work,create a new task for this local work, named MapredLocalTask. + * then make this new generated task depends on current task's parent task, and make current task + * depends on this new generated task + */ + class LocalMapJoinTaskDispatcher implements Dispatcher { + + private PhysicalContext physicalContext; + + public LocalMapJoinTaskDispatcher(PhysicalContext context) { + super(); + physicalContext = context; + } + + @Override + public Object dispatch(Node nd, Stack stack, Object... nodeOutputs) + throws SemanticException { + Task currTask = (Task) nd; + + //not map reduce task or conditional task, just skip + if (!currTask.isMapRedTask() || currTask instanceof ConditionalTask) { + return null; + } + + //get current mapred work and its local work + MapredWork mapredWork = (MapredWork) currTask.getWork(); + MapredLocalWork localwork = mapredWork.getMapLocalWork(); + + + if(localwork != null){ + + //get the context info and set up the shared tmp URI + Context ctx = physicalContext.getContext(); + String tmpFileURI = ctx.getMRTmpFileURI()+Path.SEPARATOR+currTask.getId(); + localwork.setTmpFileURI(tmpFileURI); + String stageId=currTask.getId(); + localwork.setStageId(stageId); + + //create a task for this local work; right now, this local work is shared + //by the original MapredTask and this new generated MapredLocalTask. + MapredLocalTask localTask = (MapredLocalTask) TaskFactory.get(localwork, + physicalContext.getParseContext().getConf()); + + //replace the map join operator to local_map_join operator in the operator tree + //and return all the dummy parent + List> dummyOps= adjustLocalTask(localTask); + + //create new local work and setup the dummy ops + MapredLocalWork newLocalWork = new MapredLocalWork(); + newLocalWork.setDummyParentOp(dummyOps); + newLocalWork.setTmpFileURI(tmpFileURI); + newLocalWork.setStageId(stageId); + mapredWork.setMapLocalWork(newLocalWork); + + //get all parent tasks + List> parentTasks = currTask.getParentTasks(); + currTask.setParentTasks(null); + if (parentTasks != null) { + + for (Task tsk : parentTasks) { + //make new generated task depends on all the parent tasks of current task. + tsk.addDependentTask(localTask); + //remove the current task from its original parent task's dependent task + tsk.removeDependentTask(currTask); + } + + }else{ + //in this case, current task is in the root tasks + //so add this new task into root tasks and remove the current task from root tasks + physicalContext.addToRootTask(localTask); + physicalContext.removeFromRootTask(currTask); + } + + //make current task depends on this new generated localMapJoinTask + //now localTask is the parent task of the current task + localTask.addDependentTask(currTask); + + } + return null; + } + + //replace the map join operator to local_map_join operator in the operator tree + private List> adjustLocalTask(MapredLocalTask task) throws SemanticException { + + LocalMapJoinProcCtx localMapJoinProcCtx = new LocalMapJoinProcCtx(task, + physicalContext.getParseContext()); + + Map opRules = new LinkedHashMap(); + //opRules.put(new RuleRegExp("R1", "MAPJOIN%.*MAPJOIN%"), + //LocalMapJoinProcFactory.getMapJoinMapJoinProc()); + opRules.put(new RuleRegExp("R1", "MAPJOIN%"), LocalMapJoinProcFactory.getJoinProc()); + + // The dispatcher fires the processor corresponding to the closest + // matching rule and passes the context along + Dispatcher disp = new DefaultRuleDispatcher(LocalMapJoinProcFactory + .getDefaultProc(), opRules, localMapJoinProcCtx); + GraphWalker ogw = new DefaultGraphWalker(disp); + + // iterator the reducer operator tree + ArrayList topNodes = new ArrayList(); + + topNodes.addAll(task.getWork().getAliasToWork().values()); + ogw.startWalking(topNodes, null); + + return localMapJoinProcCtx.getDummyParentOp(); + + } + + public PhysicalContext getPhysicalContext() { + return physicalContext; + } + + public void setPhysicalContext(PhysicalContext physicalContext) { + this.physicalContext = physicalContext; + } + } + /** + * A container of current task and parse context. + */ + public static class LocalMapJoinProcCtx implements NodeProcessorCtx { + private Task currentTask; + private ParseContext parseCtx; + private List> dummyParentOp = null; + + public LocalMapJoinProcCtx(Task task, + ParseContext parseCtx) { + currentTask = task; + this.parseCtx = parseCtx; + dummyParentOp = new ArrayList>(); + } + + public Task getCurrentTask() { + return currentTask; + } + + public void setCurrentTask(Task currentTask) { + this.currentTask = currentTask; + } + + public ParseContext getParseCtx() { + return parseCtx; + } + + public void setParseCtx(ParseContext parseCtx) { + this.parseCtx = parseCtx; + } + + public void setDummyParentOp(List> op){ + this.dummyParentOp=op; + } + + public List> getDummyParentOp(){ + return this.dummyParentOp; + } + public void addDummyParentOp(Operator op){ + this.dummyParentOp.add(op); + } + + } +} + Property changes on: ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java ___________________________________________________________________ Added: svn:executable + * Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalContext.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalContext.java (revision 997835) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalContext.java (working copy) @@ -72,4 +72,11 @@ this.context = context; } + public void addToRootTask(Task tsk){ + rootTasks.add(tsk); + } + public void removeFromRootTask(Task tsk){ + rootTasks.remove(tsk); + } + } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java (revision 997835) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java (working copy) @@ -41,7 +41,7 @@ /** * create the list of physical plan resolvers. - * + * * @param hiveConf */ private void initialize(HiveConf hiveConf) { @@ -49,11 +49,12 @@ if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVESKEWJOIN)) { resolvers.add(new SkewJoinResolver()); } + resolvers.add(new MapJoinResolver()); } /** * invoke all the resolvers one-by-one, and alter the physical plan. - * + * * @return PhysicalContext * @throws HiveException */ Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 997835) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy) @@ -29,9 +29,9 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.TreeSet; +import java.util.Map.Entry; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; @@ -89,7 +89,6 @@ import org.apache.hadoop.hive.ql.optimizer.GenMRFileSink1; import org.apache.hadoop.hive.ql.optimizer.GenMROperator; import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext; -import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx; import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink1; import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink2; import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink3; @@ -99,6 +98,7 @@ import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils; import org.apache.hadoop.hive.ql.optimizer.MapJoinFactory; import org.apache.hadoop.hive.ql.optimizer.Optimizer; +import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx; import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalContext; import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalOptimizer; import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner; @@ -118,7 +118,6 @@ import org.apache.hadoop.hive.ql.plan.FetchWork; import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.FilterDesc; -import org.apache.hadoop.hive.ql.plan.FilterDesc.sampleDesc; import org.apache.hadoop.hive.ql.plan.ForwardDesc; import org.apache.hadoop.hive.ql.plan.GroupByDesc; import org.apache.hadoop.hive.ql.plan.JoinCondDesc; @@ -140,11 +139,12 @@ import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.ql.plan.UDTFDesc; import org.apache.hadoop.hive.ql.plan.UnionDesc; +import org.apache.hadoop.hive.ql.plan.FilterDesc.sampleDesc; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFHash; import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode; import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe; @@ -152,9 +152,9 @@ import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; @@ -5944,7 +5944,7 @@ // the operator stack. // The dispatcher generates the plan from the operator tree Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp(new String("R1"), "TS%"), new GenMRTableScan1()); + opRules.put(new RuleRegExp(new String("R1"), "TS%"), new GenMRTableScan1()); opRules.put(new RuleRegExp(new String("R2"), "TS%.*RS%"), new GenMRRedSink1()); opRules.put(new RuleRegExp(new String("R3"), "RS%.*RS%"), Index: ql/src/java/org/apache/hadoop/hive/ql/plan/JDBMDummyDesc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/JDBMDummyDesc.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/JDBMDummyDesc.java (revision 0) @@ -0,0 +1,16 @@ +package org.apache.hadoop.hive.ql.plan; + +import java.io.Serializable; + +public class JDBMDummyDesc implements Serializable { + private TableDesc tbl; + + public TableDesc getTbl() { + return tbl; + } + + public void setTbl(TableDesc tbl) { + this.tbl = tbl; + } + +} Property changes on: ql/src/java/org/apache/hadoop/hive/ql/plan/JDBMDummyDesc.java ___________________________________________________________________ Added: svn:executable + * Index: ql/src/java/org/apache/hadoop/hive/ql/plan/MapredLocalWork.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/MapredLocalWork.java (revision 997835) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/MapredLocalWork.java (working copy) @@ -20,15 +20,15 @@ import java.io.File; import java.io.Serializable; -import java.net.URI; import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map.Entry; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.exec.BucketMatcher; import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.fs.Path; /** * MapredLocalWork. @@ -42,8 +42,16 @@ private LinkedHashMap aliasToFetchWork; private boolean inputFileChangeSensitive; private BucketMapJoinContext bucketMapjoinContext; + private String tmpFileURI; + private String stageId; + + + + + private List> dummyParentOp ; public MapredLocalWork() { + } public MapredLocalWork( @@ -51,13 +59,40 @@ final LinkedHashMap aliasToFetchWork) { this.aliasToWork = aliasToWork; this.aliasToFetchWork = aliasToFetchWork; + + } + + public String getStageId() { + return stageId; + } + + public void setStageId(String stageId) { + this.stageId = stageId; + } + + public MapredLocalWork(MapredLocalWork clone){ + this.tmpFileURI = clone.tmpFileURI; + this.inputFileChangeSensitive=clone.inputFileChangeSensitive; + } + + public void setDummyParentOp(List> op){ + this.dummyParentOp=op; + } + + + public List> getDummyParentOp(){ + return this.dummyParentOp; + } + + @Explain(displayName = "Alias -> Map Local Operator Tree") public LinkedHashMap> getAliasToWork() { return aliasToWork; } + public void setAliasToWork( final LinkedHashMap> aliasToWork) { this.aliasToWork = aliasToWork; @@ -88,6 +123,8 @@ this.inputFileChangeSensitive = inputFileChangeSensitive; } + + public void deriveExplainAttributes() { if (bucketMapjoinContext != null) { bucketMapjoinContext.deriveBucketMapJoinMapping(); @@ -110,6 +147,14 @@ this.bucketMapjoinContext = bucketMapjoinContext; } + public void setTmpFileURI(String tmpFileURI) { + this.tmpFileURI = tmpFileURI; + } + + public String getTmpFileURI() { + return tmpFileURI; + } + public static class BucketMapJoinContext implements Serializable { private static final long serialVersionUID = 1L; @@ -198,11 +243,13 @@ this.aliasBucketFileNameMapping = aliasBucketFileNameMapping; } + @Override public String toString() { - if (aliasBucketFileNameMapping != null) + if (aliasBucketFileNameMapping != null) { return "Mapping:" + aliasBucketFileNameMapping.toString(); - else + } else { return ""; + } } @Explain(displayName = "Alias Bucket Base File Name Mapping", normalExplain = false) Index: ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (revision 997835) +++ ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (working copy) @@ -28,7 +28,6 @@ import java.io.FileWriter; import java.io.PrintStream; import java.io.Serializable; -import java.net.URI; import java.util.ArrayList; import java.util.Arrays; import java.util.Deque; @@ -40,11 +39,13 @@ import java.util.TreeMap; import java.util.regex.Matcher; import java.util.regex.Pattern; + import junit.framework.Test; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.MiniZooKeeperCluster; import org.apache.hadoop.hive.cli.CliDriver; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; @@ -54,6 +55,7 @@ import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.Utilities.StreamPrinter; import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat; +import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ASTNode; @@ -72,9 +74,7 @@ import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.hadoop.mapred.TextInputFormat; import org.apache.thrift.protocol.TBinaryProtocol; -import org.apache.hadoop.hbase.MiniZooKeeperCluster; import org.apache.zookeeper.ZooKeeper; -import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager; /** * QTestUtil. @@ -83,9 +83,10 @@ public class QTestUtil { private String testWarehouse; - private final String tmpdir = System.getProperty("test.tmp.dir"); + private final String tmpdir= System.getProperty("test.tmp.dir") ; private final Path tmppath = new Path(tmpdir); + private final String testFiles; private final String outDir; private final String logDir; @@ -203,7 +204,7 @@ // set fs.default.name to the uri of mini-dfs conf.setVar(HiveConf.ConfVars.HADOOPFS, dfs.getFileSystem().getUri().toString()); // hive.metastore.warehouse.dir needs to be set relative to the mini-dfs - conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, + conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, (new Path(dfs.getFileSystem().getUri().toString(), "/build/ql/test/data/warehouse/")).toString()); conf.setVar(HiveConf.ConfVars.HADOOPJT, "localhost:" + mr.getJobTrackerPort()); Index: ql/src/test/queries/clientpositive/join39.q =================================================================== --- ql/src/test/queries/clientpositive/join39.q (revision 997835) +++ ql/src/test/queries/clientpositive/join39.q (working copy) @@ -4,6 +4,11 @@ CREATE TABLE dest_j1(key STRING, value STRING, key1 string, val2 STRING) STORED AS TEXTFILE; +explain +INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(y) */ x.key, x.value, y.key, y.value +FROM src x left outer JOIN (select * from src where key <= 100) y ON (x.key = y.key); + INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(y) */ x.key, x.value, y.key, y.value Index: ql/src/test/queries/clientpositive/map_join25.q =================================================================== --- ql/src/test/queries/clientpositive/map_join25.q (revision 0) +++ ql/src/test/queries/clientpositive/map_join25.q (revision 0) @@ -0,0 +1,12 @@ +set hive.mapjoin.numrows = 2; + +CREATE TABLE dest_j1(key INT, value STRING, val2 STRING) STORED AS TEXTFILE; + +INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x) */ x.key, x.value, y.value +FROM src1 x JOIN src y ON (x.key = y.key); + +select * from dest_j1 x order by x.key; + + + Property changes on: ql/src/test/queries/clientpositive/map_join25.q ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/queries/clientpositive/map_join26.q =================================================================== --- ql/src/test/queries/clientpositive/map_join26.q (revision 0) +++ ql/src/test/queries/clientpositive/map_join26.q (revision 0) @@ -0,0 +1,11 @@ +CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE; + +INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x,y) */ x.key, z.value, y.value +FROM src1 x JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.key = z.key and z.ds='2008-04-08' and z.hr=11); + +select * from dest_j1 x order by x.key; + + + Property changes on: ql/src/test/queries/clientpositive/map_join26.q ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/queries/clientpositive/map_join27.q =================================================================== --- ql/src/test/queries/clientpositive/map_join27.q (revision 0) +++ ql/src/test/queries/clientpositive/map_join27.q (revision 0) @@ -0,0 +1,13 @@ + + +CREATE TABLE dest_j1(key INT, value STRING, val2 STRING) STORED AS TEXTFILE; + + +INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x) */ x.key, x.value, y.value +FROM src1 x JOIN src y ON (x.value = y.value); + +select * from dest_j1 x order by x.key, x.value; + + + Property changes on: ql/src/test/queries/clientpositive/map_join27.q ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/queries/clientpositive/map_join28.q =================================================================== --- ql/src/test/queries/clientpositive/map_join28.q (revision 0) +++ ql/src/test/queries/clientpositive/map_join28.q (revision 0) @@ -0,0 +1,15 @@ + + +CREATE TABLE dest_j1(key STRING, value STRING) STORED AS TEXTFILE; + +INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(z) */ subq.key1, z.value +FROM +(SELECT /*+ MAPJOIN(x) */ x.key as key1, x.value as value1, y.key as key2, y.value as value2 + FROM src1 x JOIN src y ON (x.key = y.key)) subq + JOIN srcpart z ON (subq.key1 = z.key and z.ds='2008-04-08' and z.hr=11); + +select * from dest_j1 x order by x.key; + + + Property changes on: ql/src/test/queries/clientpositive/map_join28.q ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/queries/clientpositive/map_join29.q =================================================================== --- ql/src/test/queries/clientpositive/map_join29.q (revision 0) +++ ql/src/test/queries/clientpositive/map_join29.q (revision 0) @@ -0,0 +1,8 @@ +CREATE TABLE dest_j1(key STRING, cnt1 INT, cnt2 INT); + +INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(subq1) */ subq1.key, subq1.cnt, subq2.cnt +FROM (select x.key, count(1) as cnt from src1 x group by x.key) subq1 JOIN + (select y.key, count(1) as cnt from src y group by y.key) subq2 ON (subq1.key = subq2.key); + +select * from dest_j1 x order by x.key; Property changes on: ql/src/test/queries/clientpositive/map_join29.q ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/queries/clientpositive/map_join30.q =================================================================== --- ql/src/test/queries/clientpositive/map_join30.q (revision 0) +++ ql/src/test/queries/clientpositive/map_join30.q (revision 0) @@ -0,0 +1,6 @@ +CREATE TABLE dest_j1(key INT, cnt INT); + +INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x) */ x.key, count(1) FROM src1 x JOIN src y ON (x.key = y.key) group by x.key; + +select * from dest_j1 x order by x.key; Property changes on: ql/src/test/queries/clientpositive/map_join30.q ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/queries/clientpositive/map_join31.q =================================================================== --- ql/src/test/queries/clientpositive/map_join31.q (revision 0) +++ ql/src/test/queries/clientpositive/map_join31.q (revision 0) @@ -0,0 +1,9 @@ +CREATE TABLE dest_j1(key STRING, cnt INT); + +INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(subq1) */ subq1.key, count(1) as cnt +FROM (select x.key, count(1) as cnt from src1 x group by x.key) subq1 JOIN + (select y.key, count(1) as cnt from src y group by y.key) subq2 ON (subq1.key = subq2.key) +group by subq1.key; + +select * from dest_j1 x order by x.key; Property changes on: ql/src/test/queries/clientpositive/map_join31.q ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/queries/clientpositive/map_join32.q =================================================================== --- ql/src/test/queries/clientpositive/map_join32.q (revision 0) +++ ql/src/test/queries/clientpositive/map_join32.q (revision 0) @@ -0,0 +1,11 @@ +CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE; + +INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x,z) */ x.key, z.value, y.value +FROM src1 x JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11); + +select * from dest_j1 x order by x.key; + + + Property changes on: ql/src/test/queries/clientpositive/map_join32.q ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/queries/clientpositive/map_join33.q =================================================================== --- ql/src/test/queries/clientpositive/map_join33.q (revision 0) +++ ql/src/test/queries/clientpositive/map_join33.q (revision 0) @@ -0,0 +1,11 @@ +CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE; + +INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x) */ x.key, z.value, y.value +FROM src1 x JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11); + +select * from dest_j1 x order by x.key; + + + Property changes on: ql/src/test/queries/clientpositive/map_join33.q ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/queries/clientpositive/map_join34.q =================================================================== --- ql/src/test/queries/clientpositive/map_join34.q (revision 0) +++ ql/src/test/queries/clientpositive/map_join34.q (revision 0) @@ -0,0 +1,17 @@ + + +CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE; + +INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x) */ x.key, x.value, subq1.value +FROM +( SELECT x.key as key, x.value as value from src x where x.key < 20 + UNION ALL + SELECT x1.key as key, x1.value as value from src x1 where x1.key > 100 +) subq1 +JOIN src1 x ON (x.key = subq1.key); + +select * from dest_j1 x order by x.key; + + + Property changes on: ql/src/test/queries/clientpositive/map_join34.q ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/queries/clientpositive/map_join35.q =================================================================== --- ql/src/test/queries/clientpositive/map_join35.q (revision 0) +++ ql/src/test/queries/clientpositive/map_join35.q (revision 0) @@ -0,0 +1,18 @@ + + +CREATE TABLE dest_j1(key STRING, value STRING, val2 INT) STORED AS TEXTFILE; + + +INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x) */ x.key, x.value, subq1.cnt +FROM +( SELECT x.key as key, count(1) as cnt from src x where x.key < 20 group by x.key + UNION ALL + SELECT x1.key as key, count(1) as cnt from src x1 where x1.key > 100 group by x1.key +) subq1 +JOIN src1 x ON (x.key = subq1.key); + +select * from dest_j1 x order by x.key; + + + Property changes on: ql/src/test/queries/clientpositive/map_join35.q ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/queries/clientpositive/map_join36.q =================================================================== --- ql/src/test/queries/clientpositive/map_join36.q (revision 0) +++ ql/src/test/queries/clientpositive/map_join36.q (revision 0) @@ -0,0 +1,25 @@ +set hive.mapjoin.numrows = 2; + + + + + +CREATE TABLE tmp1(key INT, cnt INT); +CREATE TABLE tmp2(key INT, cnt INT); +CREATE TABLE dest_j1(key INT, value INT, val2 INT); + +INSERT OVERWRITE TABLE tmp1 +SELECT key, count(1) from src group by key; + +INSERT OVERWRITE TABLE tmp2 +SELECT key, count(1) from src group by key; + + +INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x) */ x.key, x.cnt, y.cnt +FROM tmp1 x JOIN tmp2 y ON (x.key = y.key); + +select * from dest_j1 x order by x.key; + + + Property changes on: ql/src/test/queries/clientpositive/map_join36.q ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/queries/clientpositive/map_join37.q =================================================================== --- ql/src/test/queries/clientpositive/map_join37.q (revision 0) +++ ql/src/test/queries/clientpositive/map_join37.q (revision 0) @@ -0,0 +1,15 @@ +set hive.mapjoin.numrows = 2; + + + +CREATE TABLE dest_j1(key INT, value STRING, val2 STRING) STORED AS TEXTFILE; + + +INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(X) */ x.key, x.value, y.value +FROM src1 x JOIN src y ON (x.key = y.key); + +select * from dest_j1 x order by x.key; + + + Property changes on: ql/src/test/queries/clientpositive/map_join37.q ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/queries/clientpositive/map_join38.q =================================================================== --- ql/src/test/queries/clientpositive/map_join38.q (revision 0) +++ ql/src/test/queries/clientpositive/map_join38.q (revision 0) @@ -0,0 +1,15 @@ + + +create table tmp(col0 string, col1 string,col2 string,col3 string,col4 string,col5 string,col6 string,col7 string,col8 string,col9 string,col10 string,col11 string); + +insert overwrite table tmp select key, cast(key + 1 as int), key +2, key+3, key+4, cast(key+5 as int), key+6, key+7, key+8, key+9, key+10, cast(key+11 as int) from src where key = 100; + +select * from tmp; + + +FROM src a JOIN tmp b ON (a.key = b.col11) +SELECT /*+ MAPJOIN(a) */ a.value, b.col5, count(1) as count +where b.col11 = 111 +group by a.value, b.col5; + + Property changes on: ql/src/test/queries/clientpositive/map_join38.q ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/queries/clientpositive/map_join39.q =================================================================== --- ql/src/test/queries/clientpositive/map_join39.q (revision 0) +++ ql/src/test/queries/clientpositive/map_join39.q (revision 0) @@ -0,0 +1,15 @@ +set hive.mapjoin.cache.numrows = 2; + + + +CREATE TABLE dest_j1(key STRING, value STRING, key1 string, val2 STRING) STORED AS TEXTFILE; + + +INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(y) */ x.key, x.value, y.key, y.value +FROM src x left outer JOIN (select * from src where key <= 100) y ON (x.key = y.key); + +select * from dest_j1 x order by x.key; + + + Property changes on: ql/src/test/queries/clientpositive/map_join39.q ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/results/clientpositive/join25.q.out =================================================================== --- ql/src/test/results/clientpositive/join25.q.out (revision 997835) +++ ql/src/test/results/clientpositive/join25.q.out (working copy) @@ -1,4 +1,5 @@ PREHOOK: query: CREATE TABLE dest_j1(key INT, value STRING, val2 STRING) STORED AS TEXTFILE +: PREHOOK: type: CREATETABLE POSTHOOK: query: CREATE TABLE dest_j1(key INT, value STRING, val2 STRING) STORED AS TEXTFILE POSTHOOK: type: CREATETABLE Index: ql/src/test/results/clientpositive/map_join25.q.out =================================================================== --- ql/src/test/results/clientpositive/map_join25.q.out (revision 0) +++ ql/src/test/results/clientpositive/map_join25.q.out (revision 0) @@ -0,0 +1,70 @@ +PREHOOK: query: CREATE TABLE dest_j1(key INT, value STRING, val2 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE dest_j1(key INT, value STRING, val2 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x) */ x.key, x.value, y.value +FROM src1 x JOIN src y ON (x.key = y.key) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x) */ x.key, x.value, y.value +FROM src1 x JOIN src y ON (x.key = y.key) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@dest_j1 +POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.val2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from dest_j1 x order by x.key +PREHOOK: type: QUERY +PREHOOK: Input: default@dest_j1 +PREHOOK: Output: file:/tmp/nzhang/hive_2010-08-17_22-53-56_988_6826737012115531394/-mr-10000 +POSTHOOK: query: select * from dest_j1 x order by x.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest_j1 +POSTHOOK: Output: file:/tmp/nzhang/hive_2010-08-17_22-53-56_988_6826737012115531394/-mr-10000 +POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.val2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] +66 val_66 val_66 +98 val_98 val_98 +98 val_98 val_98 +128 val_128 +128 val_128 +128 val_128 +146 val_146 val_146 +146 val_146 val_146 +150 val_150 val_150 +213 val_213 val_213 +213 val_213 val_213 +224 val_224 +224 val_224 +238 val_238 val_238 +238 val_238 val_238 +255 val_255 val_255 +255 val_255 val_255 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +278 val_278 val_278 +278 val_278 val_278 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +369 val_369 +369 val_369 +369 val_369 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 Property changes on: ql/src/test/results/clientpositive/map_join25.q.out ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/results/clientpositive/map_join26.q.out =================================================================== --- ql/src/test/results/clientpositive/map_join26.q.out (revision 0) +++ ql/src/test/results/clientpositive/map_join26.q.out (revision 0) @@ -0,0 +1,144 @@ +PREHOOK: query: CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x,y) */ x.key, z.value, y.value +FROM src1 x JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.key = z.key and z.ds='2008-04-08' and z.hr=11) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x,y) */ x.key, z.value, y.value +FROM src1 x JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.key = z.key and z.ds='2008-04-08' and z.hr=11) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@dest_j1 +POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.val2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from dest_j1 x order by x.key +PREHOOK: type: QUERY +PREHOOK: Input: default@dest_j1 +PREHOOK: Output: file:/tmp/jsichi/hive_2010-08-26_15-58-37_050_6374384295174848949/-mr-10000 +POSTHOOK: query: select * from dest_j1 x order by x.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest_j1 +POSTHOOK: Output: file:/tmp/jsichi/hive_2010-08-26_15-58-37_050_6374384295174848949/-mr-10000 +POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.val2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] +128 val_128 val_128 +128 val_128 val_128 +128 val_128 val_128 +128 val_128 val_128 +128 val_128 val_128 +128 val_128 val_128 +128 val_128 val_128 +128 val_128 val_128 +128 val_128 val_128 +146 val_146 val_146 +146 val_146 val_146 +146 val_146 val_146 +146 val_146 val_146 +150 val_150 val_150 +213 val_213 val_213 +213 val_213 val_213 +213 val_213 val_213 +213 val_213 val_213 +224 val_224 val_224 +224 val_224 val_224 +224 val_224 val_224 +224 val_224 val_224 +238 val_238 val_238 +238 val_238 val_238 +238 val_238 val_238 +238 val_238 val_238 +255 val_255 val_255 +255 val_255 val_255 +255 val_255 val_255 +255 val_255 val_255 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +278 val_278 val_278 +278 val_278 val_278 +278 val_278 val_278 +278 val_278 val_278 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +369 val_369 val_369 +369 val_369 val_369 +369 val_369 val_369 +369 val_369 val_369 +369 val_369 val_369 +369 val_369 val_369 +369 val_369 val_369 +369 val_369 val_369 +369 val_369 val_369 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +66 val_66 val_66 +98 val_98 val_98 +98 val_98 val_98 +98 val_98 val_98 +98 val_98 val_98 Property changes on: ql/src/test/results/clientpositive/map_join26.q.out ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/results/clientpositive/map_join27.q.out =================================================================== --- ql/src/test/results/clientpositive/map_join27.q.out (revision 0) +++ ql/src/test/results/clientpositive/map_join27.q.out (revision 0) @@ -0,0 +1,74 @@ +PREHOOK: query: CREATE TABLE dest_j1(key INT, value STRING, val2 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE dest_j1(key INT, value STRING, val2 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x) */ x.key, x.value, y.value +FROM src1 x JOIN src y ON (x.value = y.value) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x) */ x.key, x.value, y.value +FROM src1 x JOIN src y ON (x.value = y.value) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@dest_j1 +POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.val2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from dest_j1 x order by x.key, x.value +PREHOOK: type: QUERY +PREHOOK: Input: default@dest_j1 +PREHOOK: Output: file:/tmp/nzhang/hive_2010-08-19_12-09-28_811_7041709552591198660/-mr-10000 +POSTHOOK: query: select * from dest_j1 x order by x.key, x.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest_j1 +POSTHOOK: Output: file:/tmp/nzhang/hive_2010-08-19_12-09-28_811_7041709552591198660/-mr-10000 +POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.val2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] +NULL val_165 val_165 +NULL val_165 val_165 +NULL val_193 val_193 +NULL val_193 val_193 +NULL val_193 val_193 +NULL val_265 val_265 +NULL val_265 val_265 +NULL val_27 val_27 +NULL val_409 val_409 +NULL val_409 val_409 +NULL val_409 val_409 +NULL val_484 val_484 +66 val_66 val_66 +98 val_98 val_98 +98 val_98 val_98 +146 val_146 val_146 +146 val_146 val_146 +150 val_150 val_150 +213 val_213 val_213 +213 val_213 val_213 +238 val_238 val_238 +238 val_238 val_238 +255 val_255 val_255 +255 val_255 val_255 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +278 val_278 val_278 +278 val_278 val_278 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 Property changes on: ql/src/test/results/clientpositive/map_join27.q.out ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/results/clientpositive/map_join28.q.out =================================================================== --- ql/src/test/results/clientpositive/map_join28.q.out (revision 0) +++ ql/src/test/results/clientpositive/map_join28.q.out (revision 0) @@ -0,0 +1,146 @@ +PREHOOK: query: CREATE TABLE dest_j1(key STRING, value STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE dest_j1(key STRING, value STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(z) */ subq.key1, z.value +FROM +(SELECT /*+ MAPJOIN(x) */ x.key as key1, x.value as value1, y.key as key2, y.value as value2 + FROM src1 x JOIN src y ON (x.key = y.key)) subq + JOIN srcpart z ON (subq.key1 = z.key and z.ds='2008-04-08' and z.hr=11) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(z) */ subq.key1, z.value +FROM +(SELECT /*+ MAPJOIN(x) */ x.key as key1, x.value as value1, y.key as key2, y.value as value2 + FROM src1 x JOIN src y ON (x.key = y.key)) subq + JOIN srcpart z ON (subq.key1 = z.key and z.ds='2008-04-08' and z.hr=11) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@dest_j1 +POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from dest_j1 x order by x.key +PREHOOK: type: QUERY +PREHOOK: Input: default@dest_j1 +PREHOOK: Output: file:/tmp/jsichi/hive_2010-08-26_15-58-52_535_7242516801546424528/-mr-10000 +POSTHOOK: query: select * from dest_j1 x order by x.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest_j1 +POSTHOOK: Output: file:/tmp/jsichi/hive_2010-08-26_15-58-52_535_7242516801546424528/-mr-10000 +POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] +128 val_128 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +146 val_146 +146 val_146 +146 val_146 +146 val_146 +150 val_150 +213 val_213 +213 val_213 +213 val_213 +213 val_213 +224 val_224 +224 val_224 +224 val_224 +224 val_224 +238 val_238 +238 val_238 +238 val_238 +238 val_238 +255 val_255 +255 val_255 +255 val_255 +255 val_255 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +278 val_278 +278 val_278 +278 val_278 +278 val_278 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +66 val_66 +98 val_98 +98 val_98 +98 val_98 +98 val_98 Property changes on: ql/src/test/results/clientpositive/map_join28.q.out ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/results/clientpositive/map_join29.q.out =================================================================== --- ql/src/test/results/clientpositive/map_join29.q.out (revision 0) +++ ql/src/test/results/clientpositive/map_join29.q.out (revision 0) @@ -0,0 +1,50 @@ +PREHOOK: query: CREATE TABLE dest_j1(key STRING, cnt1 INT, cnt2 INT) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE dest_j1(key STRING, cnt1 INT, cnt2 INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(subq1) */ subq1.key, subq1.cnt, subq2.cnt +FROM (select x.key, count(1) as cnt from src1 x group by x.key) subq1 JOIN + (select y.key, count(1) as cnt from src y group by y.key) subq2 ON (subq1.key = subq2.key) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(subq1) */ subq1.key, subq1.cnt, subq2.cnt +FROM (select x.key, count(1) as cnt from src1 x group by x.key) subq1 JOIN + (select y.key, count(1) as cnt from src y group by y.key) subq2 ON (subq1.key = subq2.key) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@dest_j1 +POSTHOOK: Lineage: dest_j1.cnt1 EXPRESSION [(src1)x.null, ] +POSTHOOK: Lineage: dest_j1.cnt2 EXPRESSION [(src)y.null, ] +POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: select * from dest_j1 x order by x.key +PREHOOK: type: QUERY +PREHOOK: Input: default@dest_j1 +PREHOOK: Output: file:/tmp/nzhang/hive_2010-08-17_22-54-30_633_9036208287661562782/-mr-10000 +POSTHOOK: query: select * from dest_j1 x order by x.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest_j1 +POSTHOOK: Output: file:/tmp/nzhang/hive_2010-08-17_22-54-30_633_9036208287661562782/-mr-10000 +POSTHOOK: Lineage: dest_j1.cnt1 EXPRESSION [(src1)x.null, ] +POSTHOOK: Lineage: dest_j1.cnt2 EXPRESSION [(src)y.null, ] +POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +128 1 3 +146 1 2 +150 1 1 +213 1 2 +224 1 2 +238 1 2 +255 1 2 +273 1 3 +278 1 2 +311 1 3 +369 1 3 +401 1 5 +406 1 4 +66 1 1 +98 1 2 Property changes on: ql/src/test/results/clientpositive/map_join29.q.out ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/results/clientpositive/map_join30.q.out =================================================================== --- ql/src/test/results/clientpositive/map_join30.q.out (revision 0) +++ ql/src/test/results/clientpositive/map_join30.q.out (revision 0) @@ -0,0 +1,44 @@ +PREHOOK: query: CREATE TABLE dest_j1(key INT, cnt INT) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE dest_j1(key INT, cnt INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x) */ x.key, count(1) FROM src1 x JOIN src y ON (x.key = y.key) group by x.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x) */ x.key, count(1) FROM src1 x JOIN src y ON (x.key = y.key) group by x.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@dest_j1 +POSTHOOK: Lineage: dest_j1.cnt EXPRESSION [(src1)x.null, (src)y.null, ] +POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: select * from dest_j1 x order by x.key +PREHOOK: type: QUERY +PREHOOK: Input: default@dest_j1 +PREHOOK: Output: file:/tmp/jssarma/hive_2010-07-21_11-33-19_511_4245286174894412683/10000 +POSTHOOK: query: select * from dest_j1 x order by x.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest_j1 +POSTHOOK: Output: file:/tmp/jssarma/hive_2010-07-21_11-33-19_511_4245286174894412683/10000 +POSTHOOK: Lineage: dest_j1.cnt EXPRESSION [(src1)x.null, (src)y.null, ] +POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +66 1 +98 2 +128 3 +146 2 +150 1 +213 2 +224 2 +238 2 +255 2 +273 3 +278 2 +311 3 +369 3 +401 5 +406 4 Property changes on: ql/src/test/results/clientpositive/map_join30.q.out ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/results/clientpositive/map_join31.q.out =================================================================== --- ql/src/test/results/clientpositive/map_join31.q.out (revision 0) +++ ql/src/test/results/clientpositive/map_join31.q.out (revision 0) @@ -0,0 +1,50 @@ +PREHOOK: query: CREATE TABLE dest_j1(key STRING, cnt INT) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE dest_j1(key STRING, cnt INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(subq1) */ subq1.key, count(1) as cnt +FROM (select x.key, count(1) as cnt from src1 x group by x.key) subq1 JOIN + (select y.key, count(1) as cnt from src y group by y.key) subq2 ON (subq1.key = subq2.key) +group by subq1.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(subq1) */ subq1.key, count(1) as cnt +FROM (select x.key, count(1) as cnt from src1 x group by x.key) subq1 JOIN + (select y.key, count(1) as cnt from src y group by y.key) subq2 ON (subq1.key = subq2.key) +group by subq1.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@dest_j1 +POSTHOOK: Lineage: dest_j1.cnt EXPRESSION [(src1)x.null, (src)y.null, ] +POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: select * from dest_j1 x order by x.key +PREHOOK: type: QUERY +PREHOOK: Input: default@dest_j1 +PREHOOK: Output: file:/tmp/jssarma/hive_2010-07-21_11-33-32_487_8490983650111837625/10000 +POSTHOOK: query: select * from dest_j1 x order by x.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest_j1 +POSTHOOK: Output: file:/tmp/jssarma/hive_2010-07-21_11-33-32_487_8490983650111837625/10000 +POSTHOOK: Lineage: dest_j1.cnt EXPRESSION [(src1)x.null, (src)y.null, ] +POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +128 1 +146 1 +150 1 +213 1 +224 1 +238 1 +255 1 +273 1 +278 1 +311 1 +369 1 +401 1 +406 1 +66 1 +98 1 Property changes on: ql/src/test/results/clientpositive/map_join31.q.out ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/results/clientpositive/map_join32.q.out =================================================================== --- ql/src/test/results/clientpositive/map_join32.q.out (revision 0) +++ ql/src/test/results/clientpositive/map_join32.q.out (revision 0) @@ -0,0 +1,122 @@ +PREHOOK: query: CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x,z) */ x.key, z.value, y.value +FROM src1 x JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x,z) */ x.key, z.value, y.value +FROM src1 x JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@dest_j1 +POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)y.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from dest_j1 x order by x.key +PREHOOK: type: QUERY +PREHOOK: Input: default@dest_j1 +PREHOOK: Output: file:/tmp/jsichi/hive_2010-08-26_15-59-45_924_802354274907767943/-mr-10000 +POSTHOOK: query: select * from dest_j1 x order by x.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest_j1 +POSTHOOK: Output: file:/tmp/jsichi/hive_2010-08-26_15-59-45_924_802354274907767943/-mr-10000 +POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)y.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] +146 val_146 val_146 +146 val_146 val_146 +146 val_146 val_146 +146 val_146 val_146 +150 val_150 val_150 +213 val_213 val_213 +213 val_213 val_213 +213 val_213 val_213 +213 val_213 val_213 +238 val_238 val_238 +238 val_238 val_238 +238 val_238 val_238 +238 val_238 val_238 +255 val_255 val_255 +255 val_255 val_255 +255 val_255 val_255 +255 val_255 val_255 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +278 val_278 val_278 +278 val_278 val_278 +278 val_278 val_278 +278 val_278 val_278 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +66 val_66 val_66 +98 val_98 val_98 +98 val_98 val_98 +98 val_98 val_98 +98 val_98 val_98 Property changes on: ql/src/test/results/clientpositive/map_join32.q.out ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/results/clientpositive/map_join33.q.out =================================================================== --- ql/src/test/results/clientpositive/map_join33.q.out (revision 0) +++ ql/src/test/results/clientpositive/map_join33.q.out (revision 0) @@ -0,0 +1,122 @@ +PREHOOK: query: CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x) */ x.key, z.value, y.value +FROM src1 x JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x) */ x.key, z.value, y.value +FROM src1 x JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@dest_j1 +POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)y.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from dest_j1 x order by x.key +PREHOOK: type: QUERY +PREHOOK: Input: default@dest_j1 +PREHOOK: Output: file:/tmp/jsichi/hive_2010-08-26_15-59-56_099_6500613120945442582/-mr-10000 +POSTHOOK: query: select * from dest_j1 x order by x.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest_j1 +POSTHOOK: Output: file:/tmp/jsichi/hive_2010-08-26_15-59-56_099_6500613120945442582/-mr-10000 +POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)y.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] +146 val_146 val_146 +146 val_146 val_146 +146 val_146 val_146 +146 val_146 val_146 +150 val_150 val_150 +213 val_213 val_213 +213 val_213 val_213 +213 val_213 val_213 +213 val_213 val_213 +238 val_238 val_238 +238 val_238 val_238 +238 val_238 val_238 +238 val_238 val_238 +255 val_255 val_255 +255 val_255 val_255 +255 val_255 val_255 +255 val_255 val_255 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +278 val_278 val_278 +278 val_278 val_278 +278 val_278 val_278 +278 val_278 val_278 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +66 val_66 val_66 +98 val_98 val_98 +98 val_98 val_98 +98 val_98 val_98 +98 val_98 val_98 Property changes on: ql/src/test/results/clientpositive/map_join33.q.out ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/results/clientpositive/map_join34.q.out =================================================================== --- ql/src/test/results/clientpositive/map_join34.q.out (revision 0) +++ ql/src/test/results/clientpositive/map_join34.q.out (revision 0) @@ -0,0 +1,77 @@ +PREHOOK: query: CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x) */ x.key, x.value, subq1.value +FROM +( SELECT x.key as key, x.value as value from src x where x.key < 20 + UNION ALL + SELECT x1.key as key, x1.value as value from src x1 where x1.key > 100 +) subq1 +JOIN src1 x ON (x.key = subq1.key) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x) */ x.key, x.value, subq1.value +FROM +( SELECT x.key as key, x.value as value from src x where x.key < 20 + UNION ALL + SELECT x1.key as key, x1.value as value from src x1 where x1.key > 100 +) subq1 +JOIN src1 x ON (x.key = subq1.key) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@dest_j1 +POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)x.FieldSchema(name:value, type:string, comment:default), (src)x1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from dest_j1 x order by x.key +PREHOOK: type: QUERY +PREHOOK: Input: default@dest_j1 +PREHOOK: Output: file:/tmp/nzhang/hive_2010-08-17_22-54-50_742_4903058281649790450/-mr-10000 +POSTHOOK: query: select * from dest_j1 x order by x.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest_j1 +POSTHOOK: Output: file:/tmp/nzhang/hive_2010-08-17_22-54-50_742_4903058281649790450/-mr-10000 +POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)x.FieldSchema(name:value, type:string, comment:default), (src)x1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] +128 val_128 +128 val_128 +128 val_128 +146 val_146 val_146 +146 val_146 val_146 +150 val_150 val_150 +213 val_213 val_213 +213 val_213 val_213 +224 val_224 +224 val_224 +238 val_238 val_238 +238 val_238 val_238 +255 val_255 val_255 +255 val_255 val_255 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +278 val_278 val_278 +278 val_278 val_278 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +369 val_369 +369 val_369 +369 val_369 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 Property changes on: ql/src/test/results/clientpositive/map_join34.q.out ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/results/clientpositive/map_join35.q.out =================================================================== --- ql/src/test/results/clientpositive/map_join35.q.out (revision 0) +++ ql/src/test/results/clientpositive/map_join35.q.out (revision 0) @@ -0,0 +1,56 @@ +PREHOOK: query: CREATE TABLE dest_j1(key STRING, value STRING, val2 INT) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE dest_j1(key STRING, value STRING, val2 INT) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x) */ x.key, x.value, subq1.cnt +FROM +( SELECT x.key as key, count(1) as cnt from src x where x.key < 20 group by x.key + UNION ALL + SELECT x1.key as key, count(1) as cnt from src x1 where x1.key > 100 group by x1.key +) subq1 +JOIN src1 x ON (x.key = subq1.key) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x) */ x.key, x.value, subq1.cnt +FROM +( SELECT x.key as key, count(1) as cnt from src x where x.key < 20 group by x.key + UNION ALL + SELECT x1.key as key, count(1) as cnt from src x1 where x1.key > 100 group by x1.key +) subq1 +JOIN src1 x ON (x.key = subq1.key) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@dest_j1 +POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)x.null, (src)x1.null, ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from dest_j1 x order by x.key +PREHOOK: type: QUERY +PREHOOK: Input: default@dest_j1 +PREHOOK: Output: file:/tmp/nzhang/hive_2010-08-17_22-55-11_006_4340375283672634760/-mr-10000 +POSTHOOK: query: select * from dest_j1 x order by x.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest_j1 +POSTHOOK: Output: file:/tmp/nzhang/hive_2010-08-17_22-55-11_006_4340375283672634760/-mr-10000 +POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)x.null, (src)x1.null, ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] +128 3 +146 val_146 2 +150 val_150 1 +213 val_213 2 +224 2 +238 val_238 2 +255 val_255 2 +273 val_273 3 +278 val_278 2 +311 val_311 3 +369 3 +401 val_401 5 +406 val_406 4 Property changes on: ql/src/test/results/clientpositive/map_join35.q.out ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/results/clientpositive/map_join36.q.out =================================================================== --- ql/src/test/results/clientpositive/map_join36.q.out (revision 0) +++ ql/src/test/results/clientpositive/map_join36.q.out (revision 0) @@ -0,0 +1,386 @@ +PREHOOK: query: CREATE TABLE tmp1(key INT, cnt INT) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE tmp1(key INT, cnt INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tmp1 +PREHOOK: query: CREATE TABLE tmp2(key INT, cnt INT) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE tmp2(key INT, cnt INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tmp2 +PREHOOK: query: CREATE TABLE dest_j1(key INT, value INT, val2 INT) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE dest_j1(key INT, value INT, val2 INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: INSERT OVERWRITE TABLE tmp1 +SELECT key, count(1) from src group by key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@tmp1 +POSTHOOK: query: INSERT OVERWRITE TABLE tmp1 +SELECT key, count(1) from src group by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@tmp1 +POSTHOOK: Lineage: tmp1.cnt EXPRESSION [(src)src.null, ] +POSTHOOK: Lineage: tmp1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE tmp2 +SELECT key, count(1) from src group by key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@tmp2 +POSTHOOK: query: INSERT OVERWRITE TABLE tmp2 +SELECT key, count(1) from src group by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@tmp2 +POSTHOOK: Lineage: tmp1.cnt EXPRESSION [(src)src.null, ] +POSTHOOK: Lineage: tmp1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp2.cnt EXPRESSION [(src)src.null, ] +POSTHOOK: Lineage: tmp2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x) */ x.key, x.cnt, y.cnt +FROM tmp1 x JOIN tmp2 y ON (x.key = y.key) +PREHOOK: type: QUERY +PREHOOK: Input: default@tmp1 +PREHOOK: Input: default@tmp2 +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(x) */ x.key, x.cnt, y.cnt +FROM tmp1 x JOIN tmp2 y ON (x.key = y.key) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tmp1 +POSTHOOK: Input: default@tmp2 +POSTHOOK: Output: default@dest_j1 +POSTHOOK: Lineage: dest_j1.key SIMPLE [(tmp1)x.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: dest_j1.val2 SIMPLE [(tmp2)y.FieldSchema(name:cnt, type:int, comment:null), ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(tmp1)x.FieldSchema(name:cnt, type:int, comment:null), ] +POSTHOOK: Lineage: tmp1.cnt EXPRESSION [(src)src.null, ] +POSTHOOK: Lineage: tmp1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp2.cnt EXPRESSION [(src)src.null, ] +POSTHOOK: Lineage: tmp2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: select * from dest_j1 x order by x.key +PREHOOK: type: QUERY +PREHOOK: Input: default@dest_j1 +PREHOOK: Output: file:/tmp/jsichi/hive_2010-08-26_16-00-34_763_7222853945015683789/-mr-10000 +POSTHOOK: query: select * from dest_j1 x order by x.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest_j1 +POSTHOOK: Output: file:/tmp/jsichi/hive_2010-08-26_16-00-34_763_7222853945015683789/-mr-10000 +POSTHOOK: Lineage: dest_j1.key SIMPLE [(tmp1)x.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: dest_j1.val2 SIMPLE [(tmp2)y.FieldSchema(name:cnt, type:int, comment:null), ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(tmp1)x.FieldSchema(name:cnt, type:int, comment:null), ] +POSTHOOK: Lineage: tmp1.cnt EXPRESSION [(src)src.null, ] +POSTHOOK: Lineage: tmp1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp2.cnt EXPRESSION [(src)src.null, ] +POSTHOOK: Lineage: tmp2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +0 3 3 +2 1 1 +4 1 1 +5 3 3 +8 1 1 +9 1 1 +10 1 1 +11 1 1 +12 2 2 +15 2 2 +17 1 1 +18 2 2 +19 1 1 +20 1 1 +24 2 2 +26 2 2 +27 1 1 +28 1 1 +30 1 1 +33 1 1 +34 1 1 +35 3 3 +37 2 2 +41 1 1 +42 2 2 +43 1 1 +44 1 1 +47 1 1 +51 2 2 +53 1 1 +54 1 1 +57 1 1 +58 2 2 +64 1 1 +65 1 1 +66 1 1 +67 2 2 +69 1 1 +70 3 3 +72 2 2 +74 1 1 +76 2 2 +77 1 1 +78 1 1 +80 1 1 +82 1 1 +83 2 2 +84 2 2 +85 1 1 +86 1 1 +87 1 1 +90 3 3 +92 1 1 +95 2 2 +96 1 1 +97 2 2 +98 2 2 +100 2 2 +103 2 2 +104 2 2 +105 1 1 +111 1 1 +113 2 2 +114 1 1 +116 1 1 +118 2 2 +119 3 3 +120 2 2 +125 2 2 +126 1 1 +128 3 3 +129 2 2 +131 1 1 +133 1 1 +134 2 2 +136 1 1 +137 2 2 +138 4 4 +143 1 1 +145 1 1 +146 2 2 +149 2 2 +150 1 1 +152 2 2 +153 1 1 +155 1 1 +156 1 1 +157 1 1 +158 1 1 +160 1 1 +162 1 1 +163 1 1 +164 2 2 +165 2 2 +166 1 1 +167 3 3 +168 1 1 +169 4 4 +170 1 1 +172 2 2 +174 2 2 +175 2 2 +176 2 2 +177 1 1 +178 1 1 +179 2 2 +180 1 1 +181 1 1 +183 1 1 +186 1 1 +187 3 3 +189 1 1 +190 1 1 +191 2 2 +192 1 1 +193 3 3 +194 1 1 +195 2 2 +196 1 1 +197 2 2 +199 3 3 +200 2 2 +201 1 1 +202 1 1 +203 2 2 +205 2 2 +207 2 2 +208 3 3 +209 2 2 +213 2 2 +214 1 1 +216 2 2 +217 2 2 +218 1 1 +219 2 2 +221 2 2 +222 1 1 +223 2 2 +224 2 2 +226 1 1 +228 1 1 +229 2 2 +230 5 5 +233 2 2 +235 1 1 +237 2 2 +238 2 2 +239 2 2 +241 1 1 +242 2 2 +244 1 1 +247 1 1 +248 1 1 +249 1 1 +252 1 1 +255 2 2 +256 2 2 +257 1 1 +258 1 1 +260 1 1 +262 1 1 +263 1 1 +265 2 2 +266 1 1 +272 2 2 +273 3 3 +274 1 1 +275 1 1 +277 4 4 +278 2 2 +280 2 2 +281 2 2 +282 2 2 +283 1 1 +284 1 1 +285 1 1 +286 1 1 +287 1 1 +288 2 2 +289 1 1 +291 1 1 +292 1 1 +296 1 1 +298 3 3 +302 1 1 +305 1 1 +306 1 1 +307 2 2 +308 1 1 +309 2 2 +310 1 1 +311 3 3 +315 1 1 +316 3 3 +317 2 2 +318 3 3 +321 2 2 +322 2 2 +323 1 1 +325 2 2 +327 3 3 +331 2 2 +332 1 1 +333 2 2 +335 1 1 +336 1 1 +338 1 1 +339 1 1 +341 1 1 +342 2 2 +344 2 2 +345 1 1 +348 5 5 +351 1 1 +353 2 2 +356 1 1 +360 1 1 +362 1 1 +364 1 1 +365 1 1 +366 1 1 +367 2 2 +368 1 1 +369 3 3 +373 1 1 +374 1 1 +375 1 1 +377 1 1 +378 1 1 +379 1 1 +382 2 2 +384 3 3 +386 1 1 +389 1 1 +392 1 1 +393 1 1 +394 1 1 +395 2 2 +396 3 3 +397 2 2 +399 2 2 +400 1 1 +401 5 5 +402 1 1 +403 3 3 +404 2 2 +406 4 4 +407 1 1 +409 3 3 +411 1 1 +413 2 2 +414 2 2 +417 3 3 +418 1 1 +419 1 1 +421 1 1 +424 2 2 +427 1 1 +429 2 2 +430 3 3 +431 3 3 +432 1 1 +435 1 1 +436 1 1 +437 1 1 +438 3 3 +439 2 2 +443 1 1 +444 1 1 +446 1 1 +448 1 1 +449 1 1 +452 1 1 +453 1 1 +454 3 3 +455 1 1 +457 1 1 +458 2 2 +459 2 2 +460 1 1 +462 2 2 +463 2 2 +466 3 3 +467 1 1 +468 4 4 +469 5 5 +470 1 1 +472 1 1 +475 1 1 +477 1 1 +478 2 2 +479 1 1 +480 3 3 +481 1 1 +482 1 1 +483 1 1 +484 1 1 +485 1 1 +487 1 1 +489 4 4 +490 1 1 +491 1 1 +492 2 2 +493 1 1 +494 1 1 +495 1 1 +496 1 1 +497 1 1 +498 3 3 Property changes on: ql/src/test/results/clientpositive/map_join36.q.out ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/results/clientpositive/map_join37.q.out =================================================================== --- ql/src/test/results/clientpositive/map_join37.q.out (revision 0) +++ ql/src/test/results/clientpositive/map_join37.q.out (revision 0) @@ -0,0 +1,70 @@ +PREHOOK: query: CREATE TABLE dest_j1(key INT, value STRING, val2 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE dest_j1(key INT, value STRING, val2 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(X) */ x.key, x.value, y.value +FROM src1 x JOIN src y ON (x.key = y.key) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(X) */ x.key, x.value, y.value +FROM src1 x JOIN src y ON (x.key = y.key) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@dest_j1 +POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.val2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from dest_j1 x order by x.key +PREHOOK: type: QUERY +PREHOOK: Input: default@dest_j1 +PREHOOK: Output: file:/tmp/nzhang/hive_2010-08-19_12-09-57_616_2049257293103293523/-mr-10000 +POSTHOOK: query: select * from dest_j1 x order by x.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest_j1 +POSTHOOK: Output: file:/tmp/nzhang/hive_2010-08-19_12-09-57_616_2049257293103293523/-mr-10000 +POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.val2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] +66 val_66 val_66 +98 val_98 val_98 +98 val_98 val_98 +128 val_128 +128 val_128 +128 val_128 +146 val_146 val_146 +146 val_146 val_146 +150 val_150 val_150 +213 val_213 val_213 +213 val_213 val_213 +224 val_224 +224 val_224 +238 val_238 val_238 +238 val_238 val_238 +255 val_255 val_255 +255 val_255 val_255 +273 val_273 val_273 +273 val_273 val_273 +273 val_273 val_273 +278 val_278 val_278 +278 val_278 val_278 +311 val_311 val_311 +311 val_311 val_311 +311 val_311 val_311 +369 val_369 +369 val_369 +369 val_369 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 Property changes on: ql/src/test/results/clientpositive/map_join37.q.out ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/results/clientpositive/map_join38.q.out =================================================================== --- ql/src/test/results/clientpositive/map_join38.q.out (revision 0) +++ ql/src/test/results/clientpositive/map_join38.q.out (revision 0) @@ -0,0 +1,76 @@ +PREHOOK: query: create table tmp(col0 string, col1 string,col2 string,col3 string,col4 string,col5 string,col6 string,col7 string,col8 string,col9 string,col10 string,col11 string) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table tmp(col0 string, col1 string,col2 string,col3 string,col4 string,col5 string,col6 string,col7 string,col8 string,col9 string,col10 string,col11 string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tmp +PREHOOK: query: insert overwrite table tmp select key, cast(key + 1 as int), key +2, key+3, key+4, cast(key+5 as int), key+6, key+7, key+8, key+9, key+10, cast(key+11 as int) from src where key = 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@tmp +POSTHOOK: query: insert overwrite table tmp select key, cast(key + 1 as int), key +2, key+3, key+4, cast(key+5 as int), key+6, key+7, key+8, key+9, key+10, cast(key+11 as int) from src where key = 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@tmp +POSTHOOK: Lineage: tmp.col0 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col1 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col10 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col11 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col2 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col4 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col5 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col6 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col7 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col8 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col9 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: select * from tmp +PREHOOK: type: QUERY +PREHOOK: Input: default@tmp +PREHOOK: Output: file:/tmp/jsichi/hive_2010-08-26_16-00-50_066_9191331330135572186/-mr-10000 +POSTHOOK: query: select * from tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tmp +POSTHOOK: Output: file:/tmp/jsichi/hive_2010-08-26_16-00-50_066_9191331330135572186/-mr-10000 +POSTHOOK: Lineage: tmp.col0 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col1 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col10 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col11 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col2 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col4 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col5 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col6 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col7 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col8 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col9 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +100 101 102.0 103.0 104.0 105 106.0 107.0 108.0 109.0 110.0 111 +100 101 102.0 103.0 104.0 105 106.0 107.0 108.0 109.0 110.0 111 +PREHOOK: query: FROM src a JOIN tmp b ON (a.key = b.col11) +SELECT /*+ MAPJOIN(a) */ a.value, b.col5, count(1) as count +where b.col11 = 111 +group by a.value, b.col5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@tmp +PREHOOK: Output: file:/tmp/jsichi/hive_2010-08-26_16-00-50_441_1604865558910370709/-mr-10000 +POSTHOOK: query: FROM src a JOIN tmp b ON (a.key = b.col11) +SELECT /*+ MAPJOIN(a) */ a.value, b.col5, count(1) as count +where b.col11 = 111 +group by a.value, b.col5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@tmp +POSTHOOK: Output: file:/tmp/jsichi/hive_2010-08-26_16-00-50_441_1604865558910370709/-mr-10000 +POSTHOOK: Lineage: tmp.col0 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col1 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col10 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col11 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col2 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col4 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col5 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col6 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col7 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col8 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col9 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +val_111 105 2 Property changes on: ql/src/test/results/clientpositive/map_join38.q.out ___________________________________________________________________ Added: svn:executable + * Index: ql/src/test/results/clientpositive/map_join39.q.out =================================================================== --- ql/src/test/results/clientpositive/map_join39.q.out (revision 0) +++ ql/src/test/results/clientpositive/map_join39.q.out (revision 0) @@ -0,0 +1,599 @@ +PREHOOK: query: CREATE TABLE dest_j1(key STRING, value STRING, key1 string, val2 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE dest_j1(key STRING, value STRING, key1 string, val2 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(y) */ x.key, x.value, y.key, y.value +FROM src x left outer JOIN (select * from src where key <= 100) y ON (x.key = y.key) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: INSERT OVERWRITE TABLE dest_j1 +SELECT /*+ MAPJOIN(y) */ x.key, x.value, y.key, y.value +FROM src x left outer JOIN (select * from src where key <= 100) y ON (x.key = y.key) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest_j1 +POSTHOOK: Lineage: dest_j1.key SIMPLE [(src)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.key1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.val2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)x.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from dest_j1 x order by x.key +PREHOOK: type: QUERY +PREHOOK: Input: default@dest_j1 +PREHOOK: Output: file:/tmp/jssarma/hive_2010-07-21_11-34-38_483_2873236764151910976/10000 +POSTHOOK: query: select * from dest_j1 x order by x.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest_j1 +POSTHOOK: Output: file:/tmp/jssarma/hive_2010-07-21_11-34-38_483_2873236764151910976/10000 +POSTHOOK: Lineage: dest_j1.key SIMPLE [(src)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.key1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.val2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)x.FieldSchema(name:value, type:string, comment:default), ] +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +0 val_0 0 val_0 +10 val_10 10 val_10 +100 val_100 100 val_100 +100 val_100 100 val_100 +100 val_100 100 val_100 +100 val_100 100 val_100 +103 val_103 NULL NULL +103 val_103 NULL NULL +104 val_104 NULL NULL +104 val_104 NULL NULL +105 val_105 NULL NULL +11 val_11 11 val_11 +111 val_111 NULL NULL +113 val_113 NULL NULL +113 val_113 NULL NULL +114 val_114 NULL NULL +116 val_116 NULL NULL +118 val_118 NULL NULL +118 val_118 NULL NULL +119 val_119 NULL NULL +119 val_119 NULL NULL +119 val_119 NULL NULL +12 val_12 12 val_12 +12 val_12 12 val_12 +12 val_12 12 val_12 +12 val_12 12 val_12 +120 val_120 NULL NULL +120 val_120 NULL NULL +125 val_125 NULL NULL +125 val_125 NULL NULL +126 val_126 NULL NULL +128 val_128 NULL NULL +128 val_128 NULL NULL +128 val_128 NULL NULL +129 val_129 NULL NULL +129 val_129 NULL NULL +131 val_131 NULL NULL +133 val_133 NULL NULL +134 val_134 NULL NULL +134 val_134 NULL NULL +136 val_136 NULL NULL +137 val_137 NULL NULL +137 val_137 NULL NULL +138 val_138 NULL NULL +138 val_138 NULL NULL +138 val_138 NULL NULL +138 val_138 NULL NULL +143 val_143 NULL NULL +145 val_145 NULL NULL +146 val_146 NULL NULL +146 val_146 NULL NULL +149 val_149 NULL NULL +149 val_149 NULL NULL +15 val_15 15 val_15 +15 val_15 15 val_15 +15 val_15 15 val_15 +15 val_15 15 val_15 +150 val_150 NULL NULL +152 val_152 NULL NULL +152 val_152 NULL NULL +153 val_153 NULL NULL +155 val_155 NULL NULL +156 val_156 NULL NULL +157 val_157 NULL NULL +158 val_158 NULL NULL +160 val_160 NULL NULL +162 val_162 NULL NULL +163 val_163 NULL NULL +164 val_164 NULL NULL +164 val_164 NULL NULL +165 val_165 NULL NULL +165 val_165 NULL NULL +166 val_166 NULL NULL +167 val_167 NULL NULL +167 val_167 NULL NULL +167 val_167 NULL NULL +168 val_168 NULL NULL +169 val_169 NULL NULL +169 val_169 NULL NULL +169 val_169 NULL NULL +169 val_169 NULL NULL +17 val_17 17 val_17 +170 val_170 NULL NULL +172 val_172 NULL NULL +172 val_172 NULL NULL +174 val_174 NULL NULL +174 val_174 NULL NULL +175 val_175 NULL NULL +175 val_175 NULL NULL +176 val_176 NULL NULL +176 val_176 NULL NULL +177 val_177 NULL NULL +178 val_178 NULL NULL +179 val_179 NULL NULL +179 val_179 NULL NULL +18 val_18 18 val_18 +18 val_18 18 val_18 +18 val_18 18 val_18 +18 val_18 18 val_18 +180 val_180 NULL NULL +181 val_181 NULL NULL +183 val_183 NULL NULL +186 val_186 NULL NULL +187 val_187 NULL NULL +187 val_187 NULL NULL +187 val_187 NULL NULL +189 val_189 NULL NULL +19 val_19 19 val_19 +190 val_190 NULL NULL +191 val_191 NULL NULL +191 val_191 NULL NULL +192 val_192 NULL NULL +193 val_193 NULL NULL +193 val_193 NULL NULL +193 val_193 NULL NULL +194 val_194 NULL NULL +195 val_195 NULL NULL +195 val_195 NULL NULL +196 val_196 NULL NULL +197 val_197 NULL NULL +197 val_197 NULL NULL +199 val_199 NULL NULL +199 val_199 NULL NULL +199 val_199 NULL NULL +2 val_2 2 val_2 +20 val_20 20 val_20 +200 val_200 NULL NULL +200 val_200 NULL NULL +201 val_201 NULL NULL +202 val_202 NULL NULL +203 val_203 NULL NULL +203 val_203 NULL NULL +205 val_205 NULL NULL +205 val_205 NULL NULL +207 val_207 NULL NULL +207 val_207 NULL NULL +208 val_208 NULL NULL +208 val_208 NULL NULL +208 val_208 NULL NULL +209 val_209 NULL NULL +209 val_209 NULL NULL +213 val_213 NULL NULL +213 val_213 NULL NULL +214 val_214 NULL NULL +216 val_216 NULL NULL +216 val_216 NULL NULL +217 val_217 NULL NULL +217 val_217 NULL NULL +218 val_218 NULL NULL +219 val_219 NULL NULL +219 val_219 NULL NULL +221 val_221 NULL NULL +221 val_221 NULL NULL +222 val_222 NULL NULL +223 val_223 NULL NULL +223 val_223 NULL NULL +224 val_224 NULL NULL +224 val_224 NULL NULL +226 val_226 NULL NULL +228 val_228 NULL NULL +229 val_229 NULL NULL +229 val_229 NULL NULL +230 val_230 NULL NULL +230 val_230 NULL NULL +230 val_230 NULL NULL +230 val_230 NULL NULL +230 val_230 NULL NULL +233 val_233 NULL NULL +233 val_233 NULL NULL +235 val_235 NULL NULL +237 val_237 NULL NULL +237 val_237 NULL NULL +238 val_238 NULL NULL +238 val_238 NULL NULL +239 val_239 NULL NULL +239 val_239 NULL NULL +24 val_24 24 val_24 +24 val_24 24 val_24 +24 val_24 24 val_24 +24 val_24 24 val_24 +241 val_241 NULL NULL +242 val_242 NULL NULL +242 val_242 NULL NULL +244 val_244 NULL NULL +247 val_247 NULL NULL +248 val_248 NULL NULL +249 val_249 NULL NULL +252 val_252 NULL NULL +255 val_255 NULL NULL +255 val_255 NULL NULL +256 val_256 NULL NULL +256 val_256 NULL NULL +257 val_257 NULL NULL +258 val_258 NULL NULL +26 val_26 26 val_26 +26 val_26 26 val_26 +26 val_26 26 val_26 +26 val_26 26 val_26 +260 val_260 NULL NULL +262 val_262 NULL NULL +263 val_263 NULL NULL +265 val_265 NULL NULL +265 val_265 NULL NULL +266 val_266 NULL NULL +27 val_27 27 val_27 +272 val_272 NULL NULL +272 val_272 NULL NULL +273 val_273 NULL NULL +273 val_273 NULL NULL +273 val_273 NULL NULL +274 val_274 NULL NULL +275 val_275 NULL NULL +277 val_277 NULL NULL +277 val_277 NULL NULL +277 val_277 NULL NULL +277 val_277 NULL NULL +278 val_278 NULL NULL +278 val_278 NULL NULL +28 val_28 28 val_28 +280 val_280 NULL NULL +280 val_280 NULL NULL +281 val_281 NULL NULL +281 val_281 NULL NULL +282 val_282 NULL NULL +282 val_282 NULL NULL +283 val_283 NULL NULL +284 val_284 NULL NULL +285 val_285 NULL NULL +286 val_286 NULL NULL +287 val_287 NULL NULL +288 val_288 NULL NULL +288 val_288 NULL NULL +289 val_289 NULL NULL +291 val_291 NULL NULL +292 val_292 NULL NULL +296 val_296 NULL NULL +298 val_298 NULL NULL +298 val_298 NULL NULL +298 val_298 NULL NULL +30 val_30 30 val_30 +302 val_302 NULL NULL +305 val_305 NULL NULL +306 val_306 NULL NULL +307 val_307 NULL NULL +307 val_307 NULL NULL +308 val_308 NULL NULL +309 val_309 NULL NULL +309 val_309 NULL NULL +310 val_310 NULL NULL +311 val_311 NULL NULL +311 val_311 NULL NULL +311 val_311 NULL NULL +315 val_315 NULL NULL +316 val_316 NULL NULL +316 val_316 NULL NULL +316 val_316 NULL NULL +317 val_317 NULL NULL +317 val_317 NULL NULL +318 val_318 NULL NULL +318 val_318 NULL NULL +318 val_318 NULL NULL +321 val_321 NULL NULL +321 val_321 NULL NULL +322 val_322 NULL NULL +322 val_322 NULL NULL +323 val_323 NULL NULL +325 val_325 NULL NULL +325 val_325 NULL NULL +327 val_327 NULL NULL +327 val_327 NULL NULL +327 val_327 NULL NULL +33 val_33 33 val_33 +331 val_331 NULL NULL +331 val_331 NULL NULL +332 val_332 NULL NULL +333 val_333 NULL NULL +333 val_333 NULL NULL +335 val_335 NULL NULL +336 val_336 NULL NULL +338 val_338 NULL NULL +339 val_339 NULL NULL +34 val_34 34 val_34 +341 val_341 NULL NULL +342 val_342 NULL NULL +342 val_342 NULL NULL +344 val_344 NULL NULL +344 val_344 NULL NULL +345 val_345 NULL NULL +348 val_348 NULL NULL +348 val_348 NULL NULL +348 val_348 NULL NULL +348 val_348 NULL NULL +348 val_348 NULL NULL +35 val_35 35 val_35 +35 val_35 35 val_35 +35 val_35 35 val_35 +35 val_35 35 val_35 +35 val_35 35 val_35 +35 val_35 35 val_35 +35 val_35 35 val_35 +35 val_35 35 val_35 +35 val_35 35 val_35 +351 val_351 NULL NULL +353 val_353 NULL NULL +353 val_353 NULL NULL +356 val_356 NULL NULL +360 val_360 NULL NULL +362 val_362 NULL NULL +364 val_364 NULL NULL +365 val_365 NULL NULL +366 val_366 NULL NULL +367 val_367 NULL NULL +367 val_367 NULL NULL +368 val_368 NULL NULL +369 val_369 NULL NULL +369 val_369 NULL NULL +369 val_369 NULL NULL +37 val_37 37 val_37 +37 val_37 37 val_37 +37 val_37 37 val_37 +37 val_37 37 val_37 +373 val_373 NULL NULL +374 val_374 NULL NULL +375 val_375 NULL NULL +377 val_377 NULL NULL +378 val_378 NULL NULL +379 val_379 NULL NULL +382 val_382 NULL NULL +382 val_382 NULL NULL +384 val_384 NULL NULL +384 val_384 NULL NULL +384 val_384 NULL NULL +386 val_386 NULL NULL +389 val_389 NULL NULL +392 val_392 NULL NULL +393 val_393 NULL NULL +394 val_394 NULL NULL +395 val_395 NULL NULL +395 val_395 NULL NULL +396 val_396 NULL NULL +396 val_396 NULL NULL +396 val_396 NULL NULL +397 val_397 NULL NULL +397 val_397 NULL NULL +399 val_399 NULL NULL +399 val_399 NULL NULL +4 val_4 4 val_4 +400 val_400 NULL NULL +401 val_401 NULL NULL +401 val_401 NULL NULL +401 val_401 NULL NULL +401 val_401 NULL NULL +401 val_401 NULL NULL +402 val_402 NULL NULL +403 val_403 NULL NULL +403 val_403 NULL NULL +403 val_403 NULL NULL +404 val_404 NULL NULL +404 val_404 NULL NULL +406 val_406 NULL NULL +406 val_406 NULL NULL +406 val_406 NULL NULL +406 val_406 NULL NULL +407 val_407 NULL NULL +409 val_409 NULL NULL +409 val_409 NULL NULL +409 val_409 NULL NULL +41 val_41 41 val_41 +411 val_411 NULL NULL +413 val_413 NULL NULL +413 val_413 NULL NULL +414 val_414 NULL NULL +414 val_414 NULL NULL +417 val_417 NULL NULL +417 val_417 NULL NULL +417 val_417 NULL NULL +418 val_418 NULL NULL +419 val_419 NULL NULL +42 val_42 42 val_42 +42 val_42 42 val_42 +42 val_42 42 val_42 +42 val_42 42 val_42 +421 val_421 NULL NULL +424 val_424 NULL NULL +424 val_424 NULL NULL +427 val_427 NULL NULL +429 val_429 NULL NULL +429 val_429 NULL NULL +43 val_43 43 val_43 +430 val_430 NULL NULL +430 val_430 NULL NULL +430 val_430 NULL NULL +431 val_431 NULL NULL +431 val_431 NULL NULL +431 val_431 NULL NULL +432 val_432 NULL NULL +435 val_435 NULL NULL +436 val_436 NULL NULL +437 val_437 NULL NULL +438 val_438 NULL NULL +438 val_438 NULL NULL +438 val_438 NULL NULL +439 val_439 NULL NULL +439 val_439 NULL NULL +44 val_44 44 val_44 +443 val_443 NULL NULL +444 val_444 NULL NULL +446 val_446 NULL NULL +448 val_448 NULL NULL +449 val_449 NULL NULL +452 val_452 NULL NULL +453 val_453 NULL NULL +454 val_454 NULL NULL +454 val_454 NULL NULL +454 val_454 NULL NULL +455 val_455 NULL NULL +457 val_457 NULL NULL +458 val_458 NULL NULL +458 val_458 NULL NULL +459 val_459 NULL NULL +459 val_459 NULL NULL +460 val_460 NULL NULL +462 val_462 NULL NULL +462 val_462 NULL NULL +463 val_463 NULL NULL +463 val_463 NULL NULL +466 val_466 NULL NULL +466 val_466 NULL NULL +466 val_466 NULL NULL +467 val_467 NULL NULL +468 val_468 NULL NULL +468 val_468 NULL NULL +468 val_468 NULL NULL +468 val_468 NULL NULL +469 val_469 NULL NULL +469 val_469 NULL NULL +469 val_469 NULL NULL +469 val_469 NULL NULL +469 val_469 NULL NULL +47 val_47 47 val_47 +470 val_470 NULL NULL +472 val_472 NULL NULL +475 val_475 NULL NULL +477 val_477 NULL NULL +478 val_478 NULL NULL +478 val_478 NULL NULL +479 val_479 NULL NULL +480 val_480 NULL NULL +480 val_480 NULL NULL +480 val_480 NULL NULL +481 val_481 NULL NULL +482 val_482 NULL NULL +483 val_483 NULL NULL +484 val_484 NULL NULL +485 val_485 NULL NULL +487 val_487 NULL NULL +489 val_489 NULL NULL +489 val_489 NULL NULL +489 val_489 NULL NULL +489 val_489 NULL NULL +490 val_490 NULL NULL +491 val_491 NULL NULL +492 val_492 NULL NULL +492 val_492 NULL NULL +493 val_493 NULL NULL +494 val_494 NULL NULL +495 val_495 NULL NULL +496 val_496 NULL NULL +497 val_497 NULL NULL +498 val_498 NULL NULL +498 val_498 NULL NULL +498 val_498 NULL NULL +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +5 val_5 5 val_5 +51 val_51 51 val_51 +51 val_51 51 val_51 +51 val_51 51 val_51 +51 val_51 51 val_51 +53 val_53 53 val_53 +54 val_54 54 val_54 +57 val_57 57 val_57 +58 val_58 58 val_58 +58 val_58 58 val_58 +58 val_58 58 val_58 +58 val_58 58 val_58 +64 val_64 64 val_64 +65 val_65 65 val_65 +66 val_66 66 val_66 +67 val_67 67 val_67 +67 val_67 67 val_67 +67 val_67 67 val_67 +67 val_67 67 val_67 +69 val_69 69 val_69 +70 val_70 70 val_70 +70 val_70 70 val_70 +70 val_70 70 val_70 +70 val_70 70 val_70 +70 val_70 70 val_70 +70 val_70 70 val_70 +70 val_70 70 val_70 +70 val_70 70 val_70 +70 val_70 70 val_70 +72 val_72 72 val_72 +72 val_72 72 val_72 +72 val_72 72 val_72 +72 val_72 72 val_72 +74 val_74 74 val_74 +76 val_76 76 val_76 +76 val_76 76 val_76 +76 val_76 76 val_76 +76 val_76 76 val_76 +77 val_77 77 val_77 +78 val_78 78 val_78 +8 val_8 8 val_8 +80 val_80 80 val_80 +82 val_82 82 val_82 +83 val_83 83 val_83 +83 val_83 83 val_83 +83 val_83 83 val_83 +83 val_83 83 val_83 +84 val_84 84 val_84 +84 val_84 84 val_84 +84 val_84 84 val_84 +84 val_84 84 val_84 +85 val_85 85 val_85 +86 val_86 86 val_86 +87 val_87 87 val_87 +9 val_9 9 val_9 +90 val_90 90 val_90 +90 val_90 90 val_90 +90 val_90 90 val_90 +90 val_90 90 val_90 +90 val_90 90 val_90 +90 val_90 90 val_90 +90 val_90 90 val_90 +90 val_90 90 val_90 +90 val_90 90 val_90 +92 val_92 92 val_92 +95 val_95 95 val_95 +95 val_95 95 val_95 +95 val_95 95 val_95 +95 val_95 95 val_95 +96 val_96 96 val_96 +97 val_97 97 val_97 +97 val_97 97 val_97 +97 val_97 97 val_97 +97 val_97 97 val_97 +98 val_98 98 val_98 +98 val_98 98 val_98 +98 val_98 98 val_98 +98 val_98 98 val_98 Property changes on: ql/src/test/results/clientpositive/map_join39.q.out ___________________________________________________________________ Added: svn:executable + *