diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index e511260..cca7f3e 100644
--- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -370,7 +370,6 @@
METASTORECONNECTURLKEY("javax.jdo.option.ConnectionURL",
"jdbc:derby:;databaseName=metastore_db;create=true",
"JDBC connect string for a JDBC metastore"),
-
HMSHANDLERATTEMPTS("hive.hmshandler.retry.attempts", 1,
"The number of times to retry a HMSHandler call if there were a connection error."),
HMSHANDLERINTERVAL("hive.hmshandler.retry.interval", "1000ms",
@@ -1778,7 +1777,15 @@
"When auto reducer parallelism is enabled this factor will be used to over-partition data in shuffle edges."),
TEZ_MIN_PARTITION_FACTOR("hive.tez.min.partition.factor", 0.25f,
"When auto reducer parallelism is enabled this factor will be used to put a lower limit to the number\n" +
- "of reducers that tez specifies.")
+ "of reducers that tez specifies."),
+ TEZ_DYNAMIC_PARTITION_PRUNING(
+ "hive.tez.dynamic.partition.pruning", true,
+ "When dynamic pruning is enabled, joins on partition keys will be processed by sending events from the processing " +
+ "vertices to the tez application master. These events will be used to prune unnecessary partitions."),
+ TEZ_DYNAMIC_PARTITION_PRUNING_MAX_EVENT_SIZE("hive.tez.dynamic.partition.pruning.max.event.size", 1*1024*1024L,
+ "Maximum size of events sent by processors in dynamic pruning. If this size is crossed no pruning will take place."),
+ TEZ_DYNAMIC_PARTITION_PRUNING_MAX_DATA_SIZE("hive.tez.dynamic.parition.pruning.max.data.size", 100*1024*1024L,
+ "Maximum total data size of events in dynamic pruning.")
;
public final String varname;
diff --git data/files/agg_01-p1.txt data/files/agg_01-p1.txt
new file mode 100644
index 0000000..1e7782b
--- /dev/null
+++ data/files/agg_01-p1.txt
@@ -0,0 +1,3 @@
+1.0
+2.0
+3.0
diff --git data/files/agg_01-p2.txt data/files/agg_01-p2.txt
new file mode 100644
index 0000000..dcea843
--- /dev/null
+++ data/files/agg_01-p2.txt
@@ -0,0 +1,3 @@
+4.0
+5.0
+6.0
diff --git data/files/agg_01-p3.txt data/files/agg_01-p3.txt
new file mode 100644
index 0000000..2320c1f
--- /dev/null
+++ data/files/agg_01-p3.txt
@@ -0,0 +1,3 @@
+7.0
+8.0
+9.0
diff --git data/files/dim_shops.txt data/files/dim_shops.txt
new file mode 100644
index 0000000..66cc33f
--- /dev/null
+++ data/files/dim_shops.txt
@@ -0,0 +1,3 @@
+1,foo
+2,bar
+3,baz
diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties
index 135ecc9..0be278e 100644
--- itests/src/test/resources/testconfiguration.properties
+++ itests/src/test/resources/testconfiguration.properties
@@ -139,8 +139,11 @@ minitez.query.files.shared=alter_merge_2_orc.q,\
minitez.query.files=bucket_map_join_tez1.q,\
bucket_map_join_tez2.q,\
+ dynamic_partition_pruning.q,\
+ dynamic_partition_pruning_2.q,\
mapjoin_decimal.q,\
mrr.q,\
+ tez_bmj_schema_evolution.q,\
tez_dml.q,\
tez_fsstat.q,\
tez_insert_overwrite_local_directory_1.q,\
diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index af4a3e5..78ea21d 100644
--- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -43,7 +43,6 @@
import java.util.Arrays;
import java.util.Collection;
import java.util.Deque;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
@@ -64,10 +63,10 @@
import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
import org.apache.hadoop.hive.cli.CliDriver;
import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.common.io.CachingPrintStream;
import org.apache.hadoop.hive.common.io.DigestPrintStream;
import org.apache.hadoop.hive.common.io.SortAndDigestPrintStream;
import org.apache.hadoop.hive.common.io.SortPrintStream;
-import org.apache.hadoop.hive.common.io.CachingPrintStream;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
@@ -75,8 +74,6 @@
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.exec.vector.util.AllVectorTypesRecord;
-import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat;
import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.Table;
@@ -87,22 +84,14 @@
import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.hive.serde2.thrift.ThriftDeserializer;
-import org.apache.hadoop.hive.serde2.thrift.test.Complex;
import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.mapred.SequenceFileInputFormat;
-import org.apache.hadoop.mapred.SequenceFileOutputFormat;
-import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.util.Shell;
import org.apache.hive.common.util.StreamPrinter;
-import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.tools.ant.BuildException;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooKeeper;
-import org.junit.Assume;
import com.google.common.collect.ImmutableList;
@@ -145,8 +134,8 @@
private QTestSetup setup = null;
private boolean isSessionStateStarted = false;
- private String initScript;
- private String cleanupScript;
+ private final String initScript;
+ private final String cleanupScript;
static {
for (String srcTable : System.getProperty("test.src.tables", "").trim().split(",")) {
@@ -332,14 +321,6 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
HadoopShims shims = ShimLoader.getHadoopShims();
int numberOfDataNodes = 4;
- // can run tez tests only on hadoop 2
- if (clusterType == MiniClusterType.tez) {
- Assume.assumeTrue(ShimLoader.getMajorVersion().equals("0.23"));
- // this is necessary temporarily - there's a probem with multi datanodes on MiniTezCluster
- // will be fixed in 0.3
- numberOfDataNodes = 1;
- }
-
if (clusterType != MiniClusterType.none) {
dfs = shims.getMiniDfs(conf, numberOfDataNodes, true, null);
FileSystem fs = dfs.getFileSystem();
diff --git pom.xml pom.xml
index 8973c2b..2e128fc 100644
--- pom.xml
+++ pom.xml
@@ -150,8 +150,8 @@
1.0.1
1.7.5
4.0.4
+ 0.5.0
2.2.0
- 0.4.1-incubating
1.1
0.2
1.4
diff --git ql/if/queryplan.thrift ql/if/queryplan.thrift
index eafbe5a..6026d94 100644
--- ql/if/queryplan.thrift
+++ ql/if/queryplan.thrift
@@ -56,6 +56,7 @@ enum OperatorType {
PTF,
MUX,
DEMUX,
+ EVENT,
}
struct Operator {
diff --git ql/pom.xml ql/pom.xml
index 0729d47..c3e0adb 100644
--- ql/pom.xml
+++ ql/pom.xml
@@ -297,6 +297,38 @@
org.apache.tez
+ tez-runtime-internals
+ ${tez.version}
+ true
+
+
+ org.apache.hadoop
+ hadoop-common
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-core
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-jobclient
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-common
+
+
+ org.apache.hadoop
+ hadoop-hdfs
+
+
+ org.apache.hadoop
+ hadoop-yarn-client
+
+
+
+
+ org.apache.tez
tez-mapreduce
${tez.version}
true
diff --git ql/src/gen/thrift/gen-cpp/queryplan_types.cpp ql/src/gen/thrift/gen-cpp/queryplan_types.cpp
index 96dbb29..b9e04e2 100644
--- ql/src/gen/thrift/gen-cpp/queryplan_types.cpp
+++ ql/src/gen/thrift/gen-cpp/queryplan_types.cpp
@@ -51,7 +51,8 @@ int _kOperatorTypeValues[] = {
OperatorType::HASHTABLEDUMMY,
OperatorType::PTF,
OperatorType::MUX,
- OperatorType::DEMUX
+ OperatorType::DEMUX,
+ OperatorType::EVENT
};
const char* _kOperatorTypeNames[] = {
"JOIN",
@@ -74,9 +75,10 @@ const char* _kOperatorTypeNames[] = {
"HASHTABLEDUMMY",
"PTF",
"MUX",
- "DEMUX"
+ "DEMUX",
+ "EVENT"
};
-const std::map _OperatorType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(21, _kOperatorTypeValues, _kOperatorTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+const std::map _OperatorType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(22, _kOperatorTypeValues, _kOperatorTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
int _kTaskTypeValues[] = {
TaskType::MAP,
diff --git ql/src/gen/thrift/gen-cpp/queryplan_types.h ql/src/gen/thrift/gen-cpp/queryplan_types.h
index 634dd55..30ef711 100644
--- ql/src/gen/thrift/gen-cpp/queryplan_types.h
+++ ql/src/gen/thrift/gen-cpp/queryplan_types.h
@@ -56,7 +56,8 @@ struct OperatorType {
HASHTABLEDUMMY = 17,
PTF = 18,
MUX = 19,
- DEMUX = 20
+ DEMUX = 20,
+ EVENT = 21
};
};
diff --git ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/OperatorType.java ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/OperatorType.java
index aa094ee..6f23575 100644
--- ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/OperatorType.java
+++ ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/OperatorType.java
@@ -32,7 +32,8 @@
HASHTABLEDUMMY(17),
PTF(18),
MUX(19),
- DEMUX(20);
+ DEMUX(20),
+ EVENT(21);
private final int value;
@@ -95,6 +96,8 @@ public static OperatorType findByValue(int value) {
return MUX;
case 20:
return DEMUX;
+ case 21:
+ return EVENT;
default:
return null;
}
diff --git ql/src/gen/thrift/gen-php/Types.php ql/src/gen/thrift/gen-php/Types.php
index 5164b2c..4ed7fcc 100644
--- ql/src/gen/thrift/gen-php/Types.php
+++ ql/src/gen/thrift/gen-php/Types.php
@@ -56,6 +56,7 @@ final class OperatorType {
const PTF = 18;
const MUX = 19;
const DEMUX = 20;
+ const EVENT = 21;
static public $__names = array(
0 => 'JOIN',
1 => 'MAPJOIN',
@@ -78,6 +79,7 @@ final class OperatorType {
18 => 'PTF',
19 => 'MUX',
20 => 'DEMUX',
+ 21 => 'EVENT',
);
}
diff --git ql/src/gen/thrift/gen-py/queryplan/ttypes.py ql/src/gen/thrift/gen-py/queryplan/ttypes.py
index 2a3f745..96777fa 100644
--- ql/src/gen/thrift/gen-py/queryplan/ttypes.py
+++ ql/src/gen/thrift/gen-py/queryplan/ttypes.py
@@ -66,6 +66,7 @@ class OperatorType:
PTF = 18
MUX = 19
DEMUX = 20
+ EVENT = 21
_VALUES_TO_NAMES = {
0: "JOIN",
@@ -89,6 +90,7 @@ class OperatorType:
18: "PTF",
19: "MUX",
20: "DEMUX",
+ 21: "EVENT",
}
_NAMES_TO_VALUES = {
@@ -113,6 +115,7 @@ class OperatorType:
"PTF": 18,
"MUX": 19,
"DEMUX": 20,
+ "EVENT": 21,
}
class TaskType:
diff --git ql/src/gen/thrift/gen-rb/queryplan_types.rb ql/src/gen/thrift/gen-rb/queryplan_types.rb
index 35e1f0f..449becf 100644
--- ql/src/gen/thrift/gen-rb/queryplan_types.rb
+++ ql/src/gen/thrift/gen-rb/queryplan_types.rb
@@ -42,8 +42,9 @@ module OperatorType
PTF = 18
MUX = 19
DEMUX = 20
- VALUE_MAP = {0 => "JOIN", 1 => "MAPJOIN", 2 => "EXTRACT", 3 => "FILTER", 4 => "FORWARD", 5 => "GROUPBY", 6 => "LIMIT", 7 => "SCRIPT", 8 => "SELECT", 9 => "TABLESCAN", 10 => "FILESINK", 11 => "REDUCESINK", 12 => "UNION", 13 => "UDTF", 14 => "LATERALVIEWJOIN", 15 => "LATERALVIEWFORWARD", 16 => "HASHTABLESINK", 17 => "HASHTABLEDUMMY", 18 => "PTF", 19 => "MUX", 20 => "DEMUX"}
- VALID_VALUES = Set.new([JOIN, MAPJOIN, EXTRACT, FILTER, FORWARD, GROUPBY, LIMIT, SCRIPT, SELECT, TABLESCAN, FILESINK, REDUCESINK, UNION, UDTF, LATERALVIEWJOIN, LATERALVIEWFORWARD, HASHTABLESINK, HASHTABLEDUMMY, PTF, MUX, DEMUX]).freeze
+ EVENT = 21
+ VALUE_MAP = {0 => "JOIN", 1 => "MAPJOIN", 2 => "EXTRACT", 3 => "FILTER", 4 => "FORWARD", 5 => "GROUPBY", 6 => "LIMIT", 7 => "SCRIPT", 8 => "SELECT", 9 => "TABLESCAN", 10 => "FILESINK", 11 => "REDUCESINK", 12 => "UNION", 13 => "UDTF", 14 => "LATERALVIEWJOIN", 15 => "LATERALVIEWFORWARD", 16 => "HASHTABLESINK", 17 => "HASHTABLEDUMMY", 18 => "PTF", 19 => "MUX", 20 => "DEMUX", 21 => "EVENT"}
+ VALID_VALUES = Set.new([JOIN, MAPJOIN, EXTRACT, FILTER, FORWARD, GROUPBY, LIMIT, SCRIPT, SELECT, TABLESCAN, FILESINK, REDUCESINK, UNION, UDTF, LATERALVIEWJOIN, LATERALVIEWFORWARD, HASHTABLESINK, HASHTABLEDUMMY, PTF, MUX, DEMUX, EVENT]).freeze
end
module TaskType
diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/AppMasterEventOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/AppMasterEventOperator.java
new file mode 100644
index 0000000..7315be5
--- /dev/null
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/AppMasterEventOperator.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.ql.exec.tez.TezContext;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.AppMasterEventDesc;
+import org.apache.hadoop.hive.ql.plan.api.OperatorType;
+import org.apache.hadoop.hive.serde2.Serializer;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.tez.runtime.api.Event;
+import org.apache.tez.runtime.api.events.InputInitializerEvent;
+
+/**
+ * AppMasterEventOperator sends any rows it receives to the Tez AM. This can be
+ * used to control execution dynamically.
+ */
+@SuppressWarnings({ "deprecation", "serial" })
+public class AppMasterEventOperator extends Operator {
+
+ private transient Serializer serializer;
+ private transient DataOutputBuffer buffer;
+ private transient boolean hasReachedMaxSize = false;
+ private transient long MAX_SIZE;
+
+ @Override
+ public void initializeOp(Configuration hconf) throws HiveException {
+ MAX_SIZE = HiveConf.getLongVar(hconf, ConfVars.TEZ_DYNAMIC_PARTITION_PRUNING_MAX_EVENT_SIZE);
+ serializer =
+ (Serializer) ReflectionUtils.newInstance(conf.getTable().getDeserializerClass(), null);
+ initDataBuffer(false);
+ }
+
+ private void initDataBuffer(boolean skipPruning) throws HiveException {
+ buffer = new DataOutputBuffer();
+ try {
+ // where does this go to?
+ buffer.writeUTF(((TezContext) TezContext.get()).getTezProcessorContext().getTaskVertexName());
+
+ // add any other header info
+ getConf().writeEventHeader(buffer);
+
+ // write byte to say whether to skip pruning or not
+ buffer.writeBoolean(skipPruning);
+ } catch (IOException e) {
+ throw new HiveException(e);
+ }
+ }
+
+ @Override
+ public void processOp(Object row, int tag) throws HiveException {
+ if (hasReachedMaxSize) {
+ return;
+ }
+
+ ObjectInspector rowInspector = inputObjInspectors[0];
+ try {
+ Writable writableRow = serializer.serialize(row, rowInspector);
+ writableRow.write(buffer);
+ if (buffer.getLength() > MAX_SIZE) {
+ LOG.info("Disabling AM events. Buffer size too large: " + buffer.getLength());
+ hasReachedMaxSize = true;
+ buffer = null;
+ }
+ } catch (Exception e) {
+ throw new HiveException(e);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("AppMasterEvent: " + row);
+ }
+ forward(row, rowInspector);
+ }
+
+ @Override
+ public void closeOp(boolean abort) throws HiveException {
+ if (!abort) {
+ TezContext context = (TezContext) TezContext.get();
+
+ String vertexName = getConf().getVertexName();
+ String inputName = getConf().getInputName();
+
+ byte[] payload = null;
+
+ if (hasReachedMaxSize) {
+ initDataBuffer(true);
+ }
+
+ payload = new byte[buffer.getLength()];
+ System.arraycopy(buffer.getData(), 0, payload, 0, buffer.getLength());
+
+ Event event =
+ InputInitializerEvent.create(vertexName, inputName,
+ ByteBuffer.wrap(payload, 0, payload.length));
+
+ LOG.info("Sending Tez event to vertex = " + vertexName + ", input = " + inputName
+ + ". Payload size = " + payload.length);
+
+ context.getTezProcessorContext().sendEvents(Collections.singletonList(event));
+ }
+ }
+
+ @Override
+ public OperatorType getType() {
+ return OperatorType.EVENT;
+ }
+
+ /**
+ * @return the name of the operator
+ */
+ @Override
+ public String getName() {
+ return getOperatorName();
+ }
+
+ static public String getOperatorName() {
+ return "EVENT";
+ }
+}
diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
index af835fa..99b937c 100644
--- ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
@@ -186,7 +186,7 @@ private void loadHashTable() throws HiveException {
* process different buckets and if the container is reused to join a different bucket,
* join results can be incorrect. The cache is keyed on operator id and for bucket map join
* the operator does not change but data needed is different. For a proper fix, this
- * requires changes in the Tez API with regard to finding bucket id and
+ * requires changes in the Tez API with regard to finding bucket id and
* also ability to schedule tasks to re-use containers that have cached the specific bucket.
*/
LOG.info("This is not bucket map join, so cache");
diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java
index 2bcb481..8946221 100644
--- ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java
@@ -29,13 +29,15 @@
import org.apache.hadoop.hive.ql.exec.vector.VectorLimitOperator;
import org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator;
import org.apache.hadoop.hive.ql.exec.vector.VectorReduceSinkOperator;
-import org.apache.hadoop.hive.ql.exec.vector.VectorSelectOperator;
import org.apache.hadoop.hive.ql.exec.vector.VectorSMBMapJoinOperator;
+import org.apache.hadoop.hive.ql.exec.vector.VectorSelectOperator;
import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.AppMasterEventDesc;
import org.apache.hadoop.hive.ql.plan.CollectDesc;
import org.apache.hadoop.hive.ql.plan.DemuxDesc;
import org.apache.hadoop.hive.ql.plan.DummyStoreDesc;
+import org.apache.hadoop.hive.ql.plan.DynamicPruningEventDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExtractDesc;
import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
@@ -64,6 +66,7 @@
* OperatorFactory.
*
*/
+@SuppressWarnings({ "rawtypes", "unchecked" })
public final class OperatorFactory {
private static final List opvec;
private static final List vectorOpvec;
@@ -101,6 +104,10 @@
DemuxOperator.class));
opvec.add(new OpTuple(MuxDesc.class,
MuxOperator.class));
+ opvec.add(new OpTuple(AppMasterEventDesc.class,
+ AppMasterEventOperator.class));
+ opvec.add(new OpTuple(DynamicPruningEventDesc.class,
+ AppMasterEventOperator.class));
}
static {
@@ -119,9 +126,9 @@
private static final class OpTuple {
private final Class descClass;
- private final Class extends Operator> opClass;
+ private final Class extends Operator>> opClass;
- public OpTuple(Class descClass, Class extends Operator> opClass) {
+ public OpTuple(Class descClass, Class extends Operator>> opClass) {
this.descClass = descClass;
this.opClass = opClass;
}
diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 0c90114..ca6164a 100644
--- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -819,10 +819,12 @@ public Path read(Kryo kryo, Input input, Class type) {
}
}
- public static Set> cloneOperatorTree(Configuration conf, Set> roots) {
+ public static List> cloneOperatorTree(Configuration conf, List> roots) {
ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
serializePlan(roots, baos, conf, true);
- Set> result = deserializePlan(new ByteArrayInputStream(baos.toByteArray()),
+ @SuppressWarnings("unchecked")
+ List> result =
+ deserializePlan(new ByteArrayInputStream(baos.toByteArray()),
roots.getClass(), conf, true);
return result;
}
diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
index e83bc17..7775100 100644
--- ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
@@ -130,7 +130,7 @@ public int execute(DriverContext driverContext) {
runningViaChild = conf.getBoolVar(HiveConf.ConfVars.SUBMITVIACHILD);
- if(!runningViaChild) {
+ if (!runningViaChild) {
// we are not running this mapred task via child jvm
// so directly invoke ExecDriver
return super.execute(driverContext);
diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java
index 1f68f7b..e84e65e 100644
--- ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java
@@ -14,10 +14,10 @@
import org.apache.hadoop.hive.ql.exec.vector.VectorHashKeyWrapperBatch;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.serde2.SerDe;
-import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.ByteStream.Output;
import org.apache.hadoop.hive.serde2.ByteStream.RandomAccessOutput;
+import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.WriteBuffers;
import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
@@ -57,7 +57,7 @@
private boolean[] sortableSortOrders;
private KeyValueHelper writeHelper;
- private List