diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java index 087052b..e43d892 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java @@ -285,7 +285,7 @@ public int execute(DriverContext driverContext) { if (vectorPath) { if (validateVectorPath()) { - System.out.println("Going down the vectorization path"); + LOG.debug("Going down the vectorization path"); job.setMapperClass(VectorExecMapper.class); } else { //fall back to non-vector mode @@ -533,8 +533,7 @@ public int execute(DriverContext driverContext) { } private boolean validateVectorPath() { - System.out.println("Validating if vectorized execution is applicable"); - LOG.info("Validating if vectorized execution is applicable"); + LOG.debug("Validating if vectorized execution is applicable"); MapredWork thePlan = this.getWork(); for (String path : thePlan.getPathToPartitionInfo().keySet()) { @@ -542,9 +541,7 @@ private boolean validateVectorPath() { List> interfaceList = Arrays.asList(pd.getInputFileFormatClass().getInterfaces()); if (!interfaceList.contains(VectorizedInputFormatInterface.class)) { - System.out.println("Input format: " + pd.getInputFileFormatClassName() - + ", doesn't provide vectorized input"); - LOG.info("Input format: " + pd.getInputFileFormatClassName() + LOG.debug("Input format: " + pd.getInputFileFormatClassName() + ", doesn't provide vectorized input"); return false; } @@ -559,21 +556,18 @@ private boolean validateVectorPath() { try { vectorOp = VectorMapOperator.vectorizeOperator(op, vc); } catch (Exception e) { - LOG.info("Cannot vectorize the plan", e); - System.out.println("Cannot vectorize the plan: "+ e); + LOG.debug("Cannot vectorize the plan", e); return false; } if (vectorOp == null) { - LOG.info("Cannot vectorize the plan"); - System.out.println("Cannot vectorize the plan"); + LOG.debug("Cannot vectorize the plan"); return false; } //verify the expressions contained in the operators try { validateVectorOperator(vectorOp); } catch (HiveException e) { - LOG.info("Cannot vectorize the plan", e); - System.out.println("Cannot vectorize the plan: "+e.getMessage()); + LOG.debug("Cannot vectorize the plan", e); return false; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java index b48fee5..b7293f5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java @@ -1414,6 +1414,60 @@ public boolean supportSkewJoinOptimization() { return ret; } + /** + * Clones only the operator. The children and parent are set + * to null. + * @return Cloned operator + * @throws CloneNotSupportedException + */ + public Operator cloneOp() throws CloneNotSupportedException { + T descClone = (T) conf.clone(); + Operator ret = + (Operator) OperatorFactory.getAndMakeChild( + descClone, getSchema()); + return ret; + } + + /** + * Recursively clones all the children of the tree, + * Fixes the pointers to children, parents and the pointers to itself coming from the children. + * It does not fix the pointers to itself coming from parents, parents continue to point to + * the original child. + * @return Cloned operator + * @throws CloneNotSupportedException + */ + public Operator cloneRecursiveChildren() + throws CloneNotSupportedException { + Operator newOp = this.cloneOp(); + newOp.setParentOperators(this.parentOperators); + // Fix parent in all children + if (this.getChildOperators() == null) { + newOp.setChildOperators(null); + return newOp; + } + List> newChildren = + new ArrayList>(); + + for (Operator childOp : this.getChildOperators()) { + List> parentList = + new ArrayList>(); + for (Operator parent : childOp.getParentOperators()) { + if (parent.equals(this)) { + parentList.add(newOp); + } else { + parentList.add(parent); + } + } + // Recursively clone the children + Operator clonedChildOp = childOp.cloneRecursiveChildren(); + clonedChildOp.setParentOperators(parentList); + } + + newOp.setChildOperators(newChildren); + return newOp; + } + + /* * True only for operators which produce atmost 1 output row per input * row to it. This will allow the output column names to be directly diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java index eb52961..adbf45a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java @@ -503,7 +503,7 @@ public void setChildren(Configuration hconf) throws HiveException { op.getConf()); break; case TABLESCAN: - vectorOp = op.clone(); + vectorOp = op.cloneOp(); break; case REDUCESINK: vectorOp = new VectorReduceSinkOperator(vectorizationContext, op.getConf()); @@ -530,14 +530,18 @@ public void setChildren(Configuration hconf) throws HiveException { vectorOp.setChildOperators(vectorizedChildren); } } else { - // transfer the row-mode clients to the vectorized op parent - List> children = op.getChildOperators(); - if (children != null && !children.isEmpty()) { + // transfer the row-mode children to the vectorized op parent + List> children = + new ArrayList>(); + + if (op.getChildOperators() != null && !op.getChildOperators().isEmpty()) { List> parentList = new ArrayList>(); parentList.add(vectorOp); - for (Operator childOp : children) { - childOp.setParentOperators(parentList); + for (Operator childOp : op.getChildOperators()) { + Operator clonedOp = childOp.cloneRecursiveChildren(); + clonedOp.setParentOperators(parentList); + children.add(clonedOp); } vectorOp.setChildOperators(children); }