diff --git build-common.xml build-common.xml
index e07157f..9a0a554 100644
--- build-common.xml
+++ build-common.xml
@@ -59,7 +59,7 @@
-
+
diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 78d9cc9..187d2fa 100644
--- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -190,7 +190,6 @@
PREEXECHOOKS("hive.exec.pre.hooks", ""),
POSTEXECHOOKS("hive.exec.post.hooks", ""),
ONFAILUREHOOKS("hive.exec.failure.hooks", ""),
- OPERATORHOOKS("hive.exec.operator.hooks", ""),
CLIENTSTATSPUBLISHERS("hive.client.stats.publishers", ""),
EXECPARALLEL("hive.exec.parallel", false), // parallel query launching
EXECPARALLETHREADNUMBER("hive.exec.parallel.thread.number", 8),
@@ -565,18 +564,6 @@
HIVE_INDEX_COMPACT_QUERY_MAX_SIZE("hive.index.compact.query.max.size", (long) 10 * 1024 * 1024 * 1024), // 10G
HIVE_INDEX_COMPACT_BINARY_SEARCH("hive.index.compact.binary.search", true),
- //Profiler
- HIVEPROFILERDBCLASS("hive.profiler.dbclass","jdbc:derby"),
- HIVEPROFILERJDBCDRIVER("hive.profiler.jdbcdriver", "org.apache.derby.jdbc.EmbeddedDriver"),
- HIVEPROFILERDBCONNECTIONSTRING("hive.profiler.dbconnectionstring",
- "jdbc:derby:;databaseName=TempProfilerStore;create=true"), // automatically create database
- // default timeout for JDBC connection
- HIVE_PROFILER_JDBC_TIMEOUT("hive.profiler.jdbc.timeout", 30),
- HIVE_PROFILER_RETRIES_MAX("hive.stats.retries.max",
- 0), // maximum # of retries to insert/select/delete the stats DB
- HIVE_PROFILER_RETRIES_WAIT("hive.stats.retries.wait",
- 3000), // # milliseconds to wait before the next retry
-
// Statistics
HIVESTATSAUTOGATHER("hive.stats.autogather", true),
HIVESTATSDBCLASS("hive.stats.dbclass",
diff --git conf/hive-default.xml.template conf/hive-default.xml.template
index e266ce7..02e1cca 100644
--- conf/hive-default.xml.template
+++ conf/hive-default.xml.template
@@ -1123,42 +1123,6 @@
- hive.profiler.dbclass
- jdbc:derby
- The default database that stores temporary hive statistics.
-
-
-
- hive.profiler.jdbcdriver
- org.apache.derby.jdbc.EmbeddedDriver
- The JDBC driver for the database that stores temporary hive statistics.
-
-
-
- hive.profiler.dbconnectionstring
- jdbc:derby:;databaseName=TempStatsStore;create=true
- The default connection string for the database that stores temporary hive statistics.
-
-
-
- hive.profiler.jdbc.timeout
- 30
- Timeout value (number of seconds) used by JDBC connection and statements.
-
-
-
- hive.profiler.retries.max
- 0
- Maximum number of retries when profiler publisher/aggregator got an exception updating intermediate database. Default is no tries on failures.
-
-
-
- hive.profiler.retries.wait
- 3000
- The base waiting window (in milliseconds) before the next retry. The actual wait time is calculated by baseWindow * failues baseWindow * (failure 1) * (random number between [0.0,1.0]).
-
-
-
hive.stats.dbclass
jdbc:derby
The default database that stores temporary hive statistics.
diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java
index e5bbabf..08ffcb1 100644
--- ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java
@@ -50,7 +50,6 @@
private JobConf jc;
private boolean abort = false;
private Reporter rp;
- private List opHooks;
public static final Log l4j = LogFactory.getLog("ExecMapper");
private static boolean done;
@@ -99,7 +98,6 @@ public void configure(JobConf job) {
mo.setExecContext(execContext);
mo.initializeLocalWork(jc);
mo.initialize(jc, null);
- opHooks = OperatorHookUtils.getOperatorHooks(jc);
if (localWork == null) {
return;
@@ -132,7 +130,6 @@ public void map(Object key, Object value, OutputCollector output,
rp = reporter;
mo.setOutputCollector(oc);
mo.setReporter(rp);
- mo.setOperatorHooks(opHooks);
MapredContext.get().setReporter(reporter);
}
// reset the execContext for each new row
diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java
index a5fce39..89d9643 100644
--- ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java
@@ -66,7 +66,6 @@
private long nextCntr = 1;
private static String[] fieldNames;
- private List opHooks;
public static final Log l4j = LogFactory.getLog("ExecReducer");
private boolean isLogInfoEnabled = false;
@@ -152,7 +151,6 @@ public void configure(JobConf job) {
try {
l4j.info(reducer.dump(0));
reducer.initialize(jc, rowObjectInspector);
- opHooks = OperatorHookUtils.getOperatorHooks(jc);
} catch (Throwable e) {
abort = true;
if (e instanceof OutOfMemoryError) {
@@ -183,7 +181,6 @@ public void reduce(Object key, Iterator values, OutputCollector output,
rp = reporter;
reducer.setOutputCollector(oc);
reducer.setReporter(rp);
- reducer.setOperatorHooks(opHooks);
MapredContext.get().setReporter(reporter);
}
diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
index f885e84..6a54cbf 100644
--- ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
@@ -55,7 +55,6 @@
// Bean methods
private static final long serialVersionUID = 1L;
- List operatorHooks;
private Configuration configuration;
protected List> childOperators;
@@ -241,17 +240,6 @@ public String getIdentifier() {
return id;
}
- public void setOperatorHooks(List opHooks){
- operatorHooks = opHooks;
- if (childOperators == null) {
- return;
- }
-
- for (Operator extends OperatorDesc> op : childOperators) {
- op.setOperatorHooks(opHooks);
- }
- }
-
public void setReporter(Reporter rep) {
reporter = rep;
@@ -436,34 +424,6 @@ public void passExecContext(ExecMapperContext execContext) {
}
}
- private void enterOperatorHooks(OperatorHookContext opHookContext) throws HiveException {
- if (this.operatorHooks == null) {
- return;
- }
- for(OperatorHook opHook : this.operatorHooks) {
- opHook.enter(opHookContext);
- }
- }
-
- private void exitOperatorHooks(OperatorHookContext opHookContext) throws HiveException {
- if (this.operatorHooks == null) {
- return;
- }
- for(OperatorHook opHook : this.operatorHooks) {
- opHook.exit(opHookContext);
- }
- }
-
- private void closeOperatorHooks(OperatorHookContext opHookContext) throws HiveException {
- if (this.operatorHooks == null) {
- return;
- }
- for(OperatorHook opHook : this.operatorHooks) {
- opHook.close(opHookContext);
- }
- }
-
-
/**
* Collects all the parent's output object inspectors and calls actual
* initialization method.
@@ -536,16 +496,10 @@ public void process(Object row, int tag) throws HiveException {
}
beginTime = System.currentTimeMillis();
- OperatorHookContext opHookContext = new OperatorHookContext(this, row, tag);
- enterOperatorHooks(opHookContext);
processOp(row, tag);
- exitOperatorHooks(opHookContext);
totalTime += (System.currentTimeMillis() - beginTime);
} else {
- OperatorHookContext opHookContext = new OperatorHookContext(this, row, tag);
- enterOperatorHooks(opHookContext);
processOp(row, tag);
- exitOperatorHooks(opHookContext);
}
}
@@ -630,7 +584,6 @@ public void close(boolean abort) throws HiveException {
LOG.info(id + " forwarded " + cntr + " rows");
- closeOperatorHooks(new OperatorHookContext(this));
// call the operator specific close routine
closeOp(abort);
diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorHook.java ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorHook.java
deleted file mode 100644
index a3d49d1..0000000
--- ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorHook.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec;
-
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-/**
- * OperatorHook is a new interface which
- * executes pre/post each Operator execution in a map/reduce stage
- */
-
-public interface OperatorHook {
- public void enter(OperatorHookContext opHookContext) throws HiveException;
- public void exit(OperatorHookContext opHookContext) throws HiveException;
- public void close(OperatorHookContext opHookContext) throws HiveException;
-}
diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorHookContext.java ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorHookContext.java
deleted file mode 100644
index a27a653..0000000
--- ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorHookContext.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec;
-
-import java.util.List;
-
-public class OperatorHookContext {
- private final String operatorName;
- private final String operatorId;
- private final Object currentRow;
- private final int parentTag;
- private Operator operator;
-
- public OperatorHookContext(Operator op) {
- this(op, null, -1);
- }
-
- public OperatorHookContext(Operator op, Object row, int tag) {
- this(op.getName(), op.getIdentifier(), row, tag);
- this.operator = op;
- }
-
- private OperatorHookContext(String opName, String opId, Object row, int tag) {
- operatorName = opName;
- operatorId = opId;
- currentRow = row;
- parentTag = tag;
- }
-
- public Operator getOperator() {
- return operator;
- }
-
- public Operator getParentOperator() {
- List parents = this.operator.getParentOperators();
- if (parents == null || parents.isEmpty()) {
- return null;
- }
- return (Operator)(this.operator.getParentOperators().get(this.parentTag));
-
- }
-
- public String getOperatorName() {
- return operatorName;
- }
-
- public String getOperatorId() {
- return operatorId;
- }
-
- public Object getCurrentRow() {
- return currentRow;
- }
-
- @Override
- public String toString() {
- return "operatorName= " + this.getOperatorName() +
- ", id=" + this.getOperatorId();
-
- }
-}
diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorHookUtils.java ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorHookUtils.java
deleted file mode 100644
index d7be357..0000000
--- ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorHookUtils.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec;
-import java.util.List;
-import java.util.ArrayList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.common.JavaUtils;
-import org.apache.hadoop.hive.conf.HiveConf;
-public class OperatorHookUtils {
-
- public static final Log LOG = LogFactory.getLog("OperatorHookUtils");
- public static List getOperatorHooks(Configuration hconf)
- throws Exception {
- List opHooks = new ArrayList();
- String csOpHooks = HiveConf.getVar(hconf, HiveConf.ConfVars.OPERATORHOOKS);
- if(csOpHooks == null) {
- return opHooks;
- }
- String[] opHookClasses = csOpHooks.split(",");
- for(String opHookClass: opHookClasses) {
- try {
- OperatorHook opHook =
- (OperatorHook) Class.forName(opHookClass.trim(), true,
- JavaUtils.getClassLoader()).newInstance();
- opHooks.add(opHook);
- } catch (ClassNotFoundException e) {
- LOG.error(opHookClass + " Class not found: " + e.getMessage());
- }
- }
- return opHooks;
- }
-}
diff --git ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilePublisher.java ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilePublisher.java
deleted file mode 100644
index 6c46bac..0000000
--- ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilePublisher.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.profiler;
-
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.util.Map;
-import java.sql.SQLException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.exec.Utilities;
-
-public class HiveProfilePublisher {
- private final Log LOG = LogFactory.getLog(this.getClass().getName());
- private boolean connected = false;
- private HiveProfilePublisherInfo info;
-
-
- public boolean closeConnection() {
-
- if (info == null || info.getConnection() == null) {
- return true;
- }
- try {
- if (info.getInsertStatement() != null) {
- info.closeInsertStatement();
- }
- info.getConnection().close();
- return true;
- } catch (Exception e) {
- LOG.error("Error during JDBC termination. ", e);
- return false;
- }
- }
-
- public boolean initialize(Configuration conf) {
- try {
- info = new HiveProfilePublisherInfo(conf);
- String createTable = getCreate();
- LOG.info(createTable);
- HiveProfilerUtils.createTableIfNonExistent(info, createTable);
- info.prepareInsert();
- } catch (Exception e) {
- LOG.error("Error during HiveProfilePublisher initialization", e);
- return false;
- }
- return true;
- }
-
- private String getCreate() {
- return "CREATE TABLE " + info.getTableName() +
- " ( " +
- HiveProfilerStats.Columns.QUERY_ID + " VARCHAR(512) NOT NULL, " +
- HiveProfilerStats.Columns.TASK_ID + " VARCHAR(512) NOT NULL, " +
- HiveProfilerStats.Columns.OPERATOR_ID + " INT, " +
- HiveProfilerStats.Columns.OPERATOR_NAME + " VARCHAR(512) NOT NULL, " +
- HiveProfilerStats.Columns.PARENT_OPERATOR_ID + " INT," +
- HiveProfilerStats.Columns.PARENT_OPERATOR_NAME + " VARCHAR(512), " +
- HiveProfilerStats.Columns.LEVEL_ANNO_NAME + " VARCHAR(512), " +
- HiveProfilerStats.Columns.CALL_COUNT + " BIGINT, " +
- HiveProfilerStats.Columns.INCL_TIME + " BIGINT ) ";
-
- }
-
- public boolean publishStat(String queryId, Map stats,
- Configuration conf) {
- if (info == null || info.getConnection() == null) {
- if(!initialize(conf)) {
- return false;
- }
- } else {
- try {
- if(stats == null || stats.isEmpty()) {
- return true;
- }
- Utilities.SQLCommand execUpdate = new Utilities.SQLCommand() {
- @Override
- public Void run(PreparedStatement stmt) throws SQLException {
- stmt.executeUpdate();
- return null;
- }
- };
- PreparedStatement insStmt = info.getInsert(stats);
- Utilities.executeWithRetry(execUpdate, insStmt, info.getWaitWindow(), info.getMaxRetries());
- } catch (Exception e) {
- LOG.error("ERROR during publishing profiling data. ", e);
- return false;
- }
- }
-
- return true;
- }
-}
diff --git ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilePublisherInfo.java ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilePublisherInfo.java
deleted file mode 100644
index 0b6baa0..0000000
--- ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilePublisherInfo.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.profiler;
-
-import java.util.Map;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-import java.sql.Connection;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.exec.Utilities;
-
-public class HiveProfilePublisherInfo implements HiveProfilerConnectionInfo {
- final private Log LOG = LogFactory.getLog(this.getClass().getName());
- private String statsDbClass;
- private final String tableName = "PROFILER_STATS";
- private int maxRetries, waitWindow, timeout;
- private Connection conn;
- private String connectionString;
- private PreparedStatement insStmt;
-
- public String getDbClass() { return statsDbClass; }
-
- public int getTimeout() { return timeout; }
-
- public int getMaxRetries() { return maxRetries; }
-
- public int getWaitWindow() { return waitWindow; }
-
- public String getConnectionString() { return connectionString; }
-
- public String getTableName() { return tableName; }
-
- public Connection getConnection() { return conn; }
-
- protected PreparedStatement getInsertStatement() {
- return insStmt;
- }
-
- private String getInsert() {
- String colNames = "";
- String val = "";
- int numCols = HiveProfilerStats.COLUMN_NAMES.length;
- for (int i = 0; i < numCols; i++) {
- colNames += HiveProfilerStats.COLUMN_NAMES[i];
- val += "?";
-
- if (i < numCols - 1) {
- colNames += ",";
- val += ",";
- }
- }
- return "INSERT INTO " + tableName + " (" + colNames + ") VALUES (" + val + ")";
- }
-
- public HiveProfilePublisherInfo (Configuration conf) throws Exception{
- maxRetries = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_PROFILER_RETRIES_MAX);
- waitWindow = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_PROFILER_RETRIES_WAIT);
- connectionString = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEPROFILERDBCONNECTIONSTRING);
- timeout = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_PROFILER_JDBC_TIMEOUT);
- String driver = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEPROFILERJDBCDRIVER);
- statsDbClass = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEPROFILERDBCLASS);
- try {
- Class.forName(driver).newInstance();
- } catch (Exception e) {
- LOG.error("Error during instantiating JDBC driver " + driver + ". ", e);
- }
- DriverManager.setLoginTimeout(timeout); // stats is non-blocking
- conn = Utilities.connectWithRetry(connectionString, waitWindow, maxRetries);
- }
-
- protected void prepareInsert() throws SQLException {
- insStmt = Utilities.prepareWithRetry(conn, getInsert(), waitWindow, maxRetries);
- }
- protected void closeInsertStatement() throws SQLException {
- insStmt.close();
- }
-
- protected PreparedStatement getInsert(Map stats) throws SQLException {
- for (int i = 0; i < HiveProfilerStats.COLUMN_NAMES.length; i++) {
- insStmt.setString(i + 1, stats.get(HiveProfilerStats.COLUMN_NAMES[i]));
- }
- return insStmt;
- }
-
-}
-
diff --git ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfiler.java ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfiler.java
deleted file mode 100644
index bdc9e50..0000000
--- ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfiler.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.profiler;
-
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.ql.exec.OperatorHook;
-import org.apache.hadoop.hive.ql.exec.OperatorHookContext;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-
-public class HiveProfiler implements OperatorHook {
- private final Log LOG = LogFactory.getLog(this.getClass().getName());
- private static final HiveProfilePublisher pub = new HiveProfilePublisher();
-
- private final Map operatorCallStack =
- new ConcurrentHashMap();
-
- // Aggregates stats for each operator in memory so that stats are written to DB
- // all at once - this allows the profiler to be extremely lightweight in
- // communication with the DB
- private final Map aggrStats =
- new ConcurrentHashMap();
-
- public void enter(OperatorHookContext opHookContext) throws HiveException {
- String opLevelAnnoName = HiveProfilerUtils.getLevelAnnotatedName(opHookContext);
- HiveProfilerEntry curEntry = new HiveProfilerEntry(opHookContext);
- assert(operatorCallStack.get(opLevelAnnoName) == null);
- operatorCallStack.put(opLevelAnnoName, curEntry);
- }
-
- private void exit(HiveProfilerEntry curEntry) {
- OperatorHookContext opHookContext = curEntry.getOperatorHookContext();
- // update the metrics we are
- long exitTime = System.nanoTime();
- long wallTime = exitTime - curEntry.wallStartTime;
-
- String opName = opHookContext.getOperatorName();
-
- Configuration conf = opHookContext.getOperator().getConfiguration();
-
- String opLevelAnnoName = HiveProfilerUtils.getLevelAnnotatedName(opHookContext);
-
- if (aggrStats.containsKey(opLevelAnnoName)) {
- aggrStats.get(opLevelAnnoName).updateStats(wallTime, 1);
- } else {
- HiveProfilerStats stats =
- new HiveProfilerStats(opHookContext, 1, wallTime, conf);
- aggrStats.put(opLevelAnnoName, stats);
- }
-
- }
- public void exit(OperatorHookContext opHookContext) throws HiveException {
- if (operatorCallStack.isEmpty()) {
- LOG.error("Unexpected state: Operator Call Stack is empty on exit.");
- }
- String opLevelAnnoName = HiveProfilerUtils.getLevelAnnotatedName(opHookContext);
-
- HiveProfilerEntry curEntry = operatorCallStack.get(opLevelAnnoName);
-
- if (!curEntry.getOperatorHookContext().equals(opHookContext)) {
- LOG.error("Expected to exit from: " + curEntry.getOperatorHookContext().toString() +
- " but exit called on " + opHookContext.toString());
- }
-
- exit(curEntry);
- operatorCallStack.remove(opLevelAnnoName);
- }
-
- public void close(OperatorHookContext opHookContext) {
- Configuration conf = opHookContext.getOperator().getConfiguration();
-
- Collection stats = aggrStats.values();
- // example:
- // queryId=pamelavagata_20130115163838_4a1cb4ae-43c1-4656-bfae-118557896eec,
- // operatorName=TS,
- // id=3,
- // parentName="" (root),
- // inclTime=1202710
- // callCount
-
- Iterator statsIter = stats.iterator();
- while (statsIter.hasNext()) {
- HiveProfilerStats stat = statsIter.next();
- pub.initialize(conf);
- boolean published = pub.publishStat(null, stat.getStatsMap(), conf);
- LOG.info((published ? "did " : "did not ") + "publish stat for: " + stat.toString());
- pub.closeConnection();
- }
- stats.clear();
-
- }
-
- private class HiveProfilerEntry {
- OperatorHookContext ctxt;
- protected long wallStartTime;
-
- protected HiveProfilerEntry(OperatorHookContext opHookContext) {
- this.ctxt = opHookContext;
- this.wallStartTime = System.nanoTime();
- }
-
- protected OperatorHookContext getOperatorHookContext() {
- return ctxt;
- }
- }
-}
diff --git ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerAggregateStat.java ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerAggregateStat.java
deleted file mode 100644
index 151ea4a..0000000
--- ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerAggregateStat.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.profiler;
-
-public class HiveProfilerAggregateStat {
- long wallTime;
- long callCount;
- public HiveProfilerAggregateStat(long wallTime, long callCount){
- this.wallTime = wallTime;
- this.callCount = callCount;
- }
-
- public long getCallCount() {
- return callCount;
- }
-
- public long getWallTime() {
- return wallTime;
- }
-
- /*
- * @param wallTime: inclusive walltime in microseconds
- */
- public void update(long wallTime, long callCount) {
- this.wallTime += wallTime;
- this.callCount += callCount;
- }
-}
diff --git ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerConnectionInfo.java ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerConnectionInfo.java
deleted file mode 100644
index cd8f46d..0000000
--- ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerConnectionInfo.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.hadoop.hive.ql.profiler;
-
-import java.sql.Connection;
-public interface HiveProfilerConnectionInfo {
- public String getDbClass();
- public int getTimeout();
- public int getMaxRetries();
- public int getWaitWindow();
- public String getConnectionString();
- public String getTableName();
- public Connection getConnection();
-}
diff --git ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerStats.java ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerStats.java
deleted file mode 100644
index e4ee89c..0000000
--- ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerStats.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.profiler;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.exec.OperatorHookContext;
-import org.apache.hadoop.hive.ql.exec.Utilities;
-
-public class HiveProfilerStats {
- public final class Columns {
- public static final String QUERY_ID = "queryId";
- public static final String OPERATOR_NAME = "operatorName";
- public static final String OPERATOR_ID = "operatorId";
- public static final String PARENT_OPERATOR_ID = "parentOperatorId";
- public static final String PARENT_OPERATOR_NAME = "parentOperatorName";
- public static final String EXCL_TIME = "exclTime";
- public static final String LEVEL_ANNO_NAME = "levelAnnotatedName";
- public static final String INCL_TIME = "inclTime";
- public static final String CALL_COUNT = "callCount";
- public static final String TASK_ID = "taskId";
- }
-
- public static final String[] COLUMN_NAMES= new String[] {
- Columns.QUERY_ID,
- Columns.TASK_ID,
- Columns.OPERATOR_NAME,
- Columns.OPERATOR_ID,
- Columns.PARENT_OPERATOR_ID,
- Columns.PARENT_OPERATOR_NAME,
- Columns.LEVEL_ANNO_NAME,
- Columns.INCL_TIME,
- Columns.CALL_COUNT
- };
-
- private final Map stats = new HashMap();
-
- long callCount;
- long inclTime;
- String taskId;
-
- protected HiveProfilerStats(
- OperatorHookContext opHookContext,
- long callCount, long wallTime, Configuration conf) {
- this.callCount = callCount;
- this.inclTime = wallTime;
- this.taskId = Utilities.getTaskId(conf);
- populateStatsMap(opHookContext, conf);
- }
-
- private void populateStatsMap(OperatorHookContext opHookContext,
- Configuration conf) {
- String queryId =
- conf == null ? "no conf" : HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYID);
- stats.put(Columns.QUERY_ID, queryId);
- String opName = opHookContext.getOperatorName();
- stats.put(
- Columns.OPERATOR_NAME, opName);
- stats.put(
- Columns.OPERATOR_ID, opHookContext.getOperatorId());
-
- Operator parent = opHookContext.getParentOperator();
- String parentOpName = parent == null ? "" : parent.getName();
- stats.put(Columns.PARENT_OPERATOR_NAME, parentOpName);
-
- String parentOpId = parent == null ? "-1" : parent.getIdentifier();
- stats.put(Columns.PARENT_OPERATOR_ID, parentOpId);
-
- stats.put(Columns.LEVEL_ANNO_NAME, HiveProfilerUtils.getLevelAnnotatedName(opHookContext));
-
- }
-
- public void updateStats(long wallTime, long count) {
- this.inclTime += wallTime;
- this.callCount += count;
- }
-
- public Map getStatsMap() {
- stats.put(Columns.TASK_ID, taskId);
- stats.put(Columns.INCL_TIME, String.valueOf(inclTime));
- stats.put(Columns.CALL_COUNT, String.valueOf(callCount));
- return stats;
- }
-
- @Override
- public String toString() {
- return stats.toString();
- }
-}
-
diff --git ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerStatsAggregator.java ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerStatsAggregator.java
deleted file mode 100644
index b3233e9..0000000
--- ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerStatsAggregator.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.profiler;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.sql.ResultSet;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.exec.Utilities;
-
-public class HiveProfilerStatsAggregator {
- final private Log LOG = LogFactory.getLog(this.getClass().getName());
- private long totalTime;
- private HiveProfilePublisherInfo rawProfileConnInfo;
-
- private Map stats =
- new HashMap();
-
- public HiveProfilerStatsAggregator(HiveConf conf) {
- try {
- // initialize the raw data connection
- rawProfileConnInfo = new HiveProfilePublisherInfo(conf);
- populateAggregateStats(conf);
- } catch (Exception e) {
-
- LOG.error("Error during initialization", e);
- }
- }
-
- public long getTotalTime() {
- return totalTime;
- }
-
- public Map getAggregateStats() {
- return stats;
- }
-
-
- private void populateAggregateStats(HiveConf conf) throws SQLException {
- int waitWindow = rawProfileConnInfo.getWaitWindow();
- int maxRetries = rawProfileConnInfo.getMaxRetries();
-
- String queryId = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYID);
- String profilerStatsTable = rawProfileConnInfo.getTableName();
- String getProfileStats =
- "SELECT * FROM " + profilerStatsTable + " WHERE queryId = ? ";
- Utilities.SQLCommand execQuery = new Utilities.SQLCommand() {
- @Override
- public ResultSet run(PreparedStatement stmt) throws SQLException {
- return stmt.executeQuery();
- }
- };
-
- try {
- PreparedStatement getProfileStatsStmt =
- Utilities.prepareWithRetry(rawProfileConnInfo.getConnection(),
- getProfileStats, waitWindow, maxRetries);
- getProfileStatsStmt.setString(1, queryId);
- ResultSet result = Utilities.executeWithRetry(execQuery, getProfileStatsStmt,
- waitWindow, maxRetries);
-
- populateAggregateStats(result);
- getProfileStatsStmt.close();
- } catch(Exception e) {
- LOG.error("executing error: ", e);
- } finally {
- HiveProfilerUtils.closeConnection(rawProfileConnInfo);
- }
- }
-
- private void populateAggregateStats(ResultSet result) {
- try {
- while(result.next()){
- // string denoting parent==>child
- // example:SEL_2==>GBY_1
- String levelAnnoName = result.getString(HiveProfilerStats.Columns.LEVEL_ANNO_NAME);
- // Microseconds
- Long curInclTime = result.getLong(HiveProfilerStats.Columns.INCL_TIME) / 1000;
- Long curCallCount = result.getLong(HiveProfilerStats.Columns.CALL_COUNT);
- totalTime += curInclTime;
- if(curInclTime != null && curCallCount != null) {
- HiveProfilerAggregateStat curStat;
- if (stats.containsKey(levelAnnoName)) {
- curStat = stats.get(levelAnnoName);
- curStat.update(curInclTime, curCallCount);
- } else {
- curStat = new HiveProfilerAggregateStat(curInclTime, curCallCount);
- }
- stats.put(levelAnnoName, curStat);
- }
- }
- } catch (Exception e) {
- LOG.error("Error Aggregating Stats", e);
- }
- }
-
-}
-
diff --git ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerUtils.java ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerUtils.java
deleted file mode 100644
index d220ed7..0000000
--- ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerUtils.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.hadoop.hive.ql.profiler;
-
-import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-
-import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.exec.MapOperator;
-import org.apache.hadoop.hive.ql.exec.OperatorHookContext;
-
-public class HiveProfilerUtils {
- public static void createTableIfNonExistent(HiveProfilerConnectionInfo info,
- String createTable) throws Exception {
- Connection conn = info.getConnection();
- Statement stmt = conn.createStatement();
- stmt.setQueryTimeout(info.getTimeout());
- DatabaseMetaData dbm = conn.getMetaData();
- ResultSet rs = dbm.getTables(null, null, info.getTableName(), null);
- boolean tblExists = rs.next();
- if(!tblExists) {
- stmt.executeUpdate(createTable);
- stmt.close();
- }
- }
-
- public static boolean closeConnection(HiveProfilerConnectionInfo info) throws SQLException{
- info.getConnection().close();
- // In case of derby, explicitly shutdown the database otherwise it reports error when
- // trying to connect to the same JDBC connection string again.
- if (info.getDbClass().equalsIgnoreCase("jdbc:derby")) {
- try {
- // The following closes the derby connection. It throws an exception that has to be caught
- // and ignored.
- DriverManager.getConnection(info.getConnectionString() + ";shutdown=true");
- } catch (Exception e) {
- // Do nothing because we know that an exception is thrown anyway.
- }
- }
- return true;
- }
-
- public static String getLevelAnnotatedName(OperatorHookContext opHookContext) {
- Operator parent = opHookContext.getParentOperator();
- if (parent != null && parent instanceof MapOperator) {
- parent = null;
- }
- Operator op = opHookContext.getOperator();
- String parentOpName = parent == null ? "" : parent.getName();
- String parentOpId = parent == null ? "main()" : parent.getOperatorId();
- String levelAnnoName = parentOpId + " ==> " + op.getOperatorId();
- return levelAnnoName;
- }
-}
diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/TstOperatorHook.java ql/src/test/org/apache/hadoop/hive/ql/exec/TstOperatorHook.java
deleted file mode 100644
index e2f318b..0000000
--- ql/src/test/org/apache/hadoop/hive/ql/exec/TstOperatorHook.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.exec;
-import java.util.HashMap;
-
-import java.util.ArrayList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-public class TstOperatorHook implements OperatorHook {
- protected transient Log LOG = LogFactory.getLog(this.getClass().getName());
- private long enters=0;
- private long exits=0;
- public void enter(OperatorHookContext opHookContext) {
- enters++;
- }
-
- public void exit(OperatorHookContext opHookContext) {
- exits++;
- }
-
- public void close(OperatorHookContext opHookContext) {
- incrCounter(TstOperatorHookUtils.TEST_OPERATOR_HOOK_ENTER, opHookContext, enters);
- incrCounter(TstOperatorHookUtils.TEST_OPERATOR_HOOK_EXIT, opHookContext, exits);
- }
-
- private void incrCounter(String ctrName, OperatorHookContext opHookContext, long incrVal) {
- TstOperatorHookUtils.TestOperatorHookCounter ctr =
- TstOperatorHookUtils.TestOperatorHookCounter.valueOf(ctrName);
- Operator op = opHookContext.getOperator();
- LOG.info(ctrName);
- op.reporter.incrCounter(ctr, incrVal);
- Long val = op.reporter.getCounter(ctr).getValue();
- LOG.info(ctrName + " " + String.valueOf(val));
- }
-}
-
diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/TstOperatorHookUtils.java ql/src/test/org/apache/hadoop/hive/ql/exec/TstOperatorHookUtils.java
deleted file mode 100644
index 548336a..0000000
--- ql/src/test/org/apache/hadoop/hive/ql/exec/TstOperatorHookUtils.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec;
-
-import java.util.Collection;
-import java.io.Serializable;
-
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
-
-public class TstOperatorHookUtils {
- public static final String TEST_OPERATOR_HOOK_ENTER = "TEST_OPERATOR_HOOK_ENTER";
- public static final String TEST_OPERATOR_HOOK_EXIT = "TEST_OPERATOR_HOOK_EXIT";
-
- public static enum TestOperatorHookCounter {
- TEST_OPERATOR_HOOK_ENTER,
- TEST_OPERATOR_HOOK_EXIT,
- }
-}
diff --git ql/src/test/org/apache/hadoop/hive/ql/hooks/HiveProfilerResultsHook.java ql/src/test/org/apache/hadoop/hive/ql/hooks/HiveProfilerResultsHook.java
deleted file mode 100644
index a3df0cd..0000000
--- ql/src/test/org/apache/hadoop/hive/ql/hooks/HiveProfilerResultsHook.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.hooks;
-import java.util.Map;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import java.sql.Connection;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.profiler.HiveProfilerStatsAggregator;
-import org.apache.hadoop.hive.ql.profiler.HiveProfilerAggregateStat;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import java.sql.DriverManager;
-import org.apache.hadoop.hive.ql.exec.Utilities;
-
-public class HiveProfilerResultsHook implements ExecuteWithHookContext {
- final private Log LOG = LogFactory.getLog(this.getClass().getName());
- @Override
- public void run (HookContext hookContext) {
- SessionState sess = SessionState.get();
- HiveConf conf = sess.getConf();
- HiveProfilerStatsAggregator aggr = new HiveProfilerStatsAggregator(conf);
- Map stats = aggr.getAggregateStats();
- for (String name: stats.keySet()){
- SessionState.getConsole().printError(name + ": " + stats.get(name).getCallCount());
- }
- }
-}
diff --git ql/src/test/org/apache/hadoop/hive/ql/hooks/PostTestOperatorHook.java ql/src/test/org/apache/hadoop/hive/ql/hooks/PostTestOperatorHook.java
deleted file mode 100644
index b931209..0000000
--- ql/src/test/org/apache/hadoop/hive/ql/hooks/PostTestOperatorHook.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.hooks;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.Iterator;
-import java.io.Serializable;
-import java.io.IOException;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.exec.Task;
-import org.apache.hadoop.hive.ql.exec.TaskRunner;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
-import org.apache.hadoop.mapred.Counters;
-
-public class PostTestOperatorHook implements ExecuteWithHookContext {
- private void logEnterExitCounters(Task extends Serializable> task) throws IOException {
- if(task.getTaskHandle() != null) {
- Counters counters = task.getTaskHandle().getCounters();
- if(counters != null) {
- logCounterValue(counters, "TEST_OPERATOR_HOOK_");
- } else {
- SessionState.getConsole().printError("counters are null");
- }
- } else {
- SessionState.getConsole().printError("task handle is null");
- }
- }
-
- private void logCounterValue(Counters ctr, String name) {
- Collection counterGroups = ctr.getGroupNames();
- for (String groupName : counterGroups) {
- Counters.Group group = ctr.getGroup(groupName);
- Iterator it = group.iterator();
- while (it.hasNext()) {
- Counters.Counter counter = it.next();
- if(counter.getName().contains(name)) {
- SessionState.getConsole().printError(counter.getName() + ": " + counter.getValue());
- }
- }
- }
- }
-
- public void run(HookContext hookContext) {
- HiveConf conf = hookContext.getConf();
- List completedTasks = hookContext.getCompleteTaskList();
- if (completedTasks != null) {
- for (TaskRunner taskRunner : completedTasks) {
- Task extends Serializable> task = taskRunner.getTask();
- if (task.isMapRedTask() && !task.isMapRedLocalTask()) {
- try {
- logEnterExitCounters(task);
-
- } catch (Exception e) {
- SessionState.getConsole().printError("Error in get counters: " + e.toString());
- }
- }
- }
- }
- }
-}
diff --git ql/src/test/queries/clientpositive/hiveprofiler0.q ql/src/test/queries/clientpositive/hiveprofiler0.q
deleted file mode 100644
index 23996ec..0000000
--- ql/src/test/queries/clientpositive/hiveprofiler0.q
+++ /dev/null
@@ -1,6 +0,0 @@
-set hive.exec.operator.hooks=org.apache.hadoop.hive.ql.profiler.HiveProfiler;
-set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.HiveProfilerResultsHook;
-SET hive.task.progress=true;
-
-select count(1) from src;
-explain select count(1) from src;
diff --git ql/src/test/queries/clientpositive/hiveprofiler_script0.q ql/src/test/queries/clientpositive/hiveprofiler_script0.q
deleted file mode 100644
index 3ee30d3..0000000
--- ql/src/test/queries/clientpositive/hiveprofiler_script0.q
+++ /dev/null
@@ -1,15 +0,0 @@
-SET hive.task.progress=true;
-set hive.exec.operator.hooks=org.apache.hadoop.hive.ql.profiler.HiveProfiler;
-set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.HiveProfilerResultsHook;
-ADD FILE src/test/scripts/testgrep;
-
-
--- checking that script operator does not cause NPE
--- Derby strangeness is causing the output collector for the Hive Profiler to not get output during DB read
-
-SELECT TRANSFORM(src.key, src.value)
- USING 'testgrep' AS (tkey, tvalue)
-FROM src
-
-
-
diff --git ql/src/test/queries/clientpositive/hiveprofiler_union0.q ql/src/test/queries/clientpositive/hiveprofiler_union0.q
deleted file mode 100644
index 0a94171..0000000
--- ql/src/test/queries/clientpositive/hiveprofiler_union0.q
+++ /dev/null
@@ -1,19 +0,0 @@
-set hive.exec.operator.hooks=org.apache.hadoop.hive.ql.profiler.HiveProfiler;
-set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.HiveProfilerResultsHook;
-SET hive.task.progress=true;
-FROM (
- FROM src select src.key, src.value WHERE src.key < 100
- UNION ALL
- FROM src SELECT src.* WHERE src.key > 100
-) unioninput
-SELECT unioninput.*;
-
-explain
- FROM (
- FROM src select src.key, src.value WHERE src.key < 100
- UNION ALL
- FROM src SELECT src.* WHERE src.key > 100
- ) unioninput
- SELECT unioninput.*;
-
-
diff --git ql/src/test/queries/clientpositive/operatorhook.q ql/src/test/queries/clientpositive/operatorhook.q
deleted file mode 100644
index c29ac2e..0000000
--- ql/src/test/queries/clientpositive/operatorhook.q
+++ /dev/null
@@ -1,6 +0,0 @@
-SET hive.exec.operator.hooks=org.apache.hadoop.hive.ql.exec.TstOperatorHook;
-SET hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostTestOperatorHook;
-SET hive.exec.mode.local.auto=false;
-SET hive.task.progress=true;
-
-SELECT count(1) FROM src
diff --git ql/src/test/results/clientpositive/hiveprofiler0.q.out ql/src/test/results/clientpositive/hiveprofiler0.q.out
deleted file mode 100644
index d1777e2..0000000
--- ql/src/test/results/clientpositive/hiveprofiler0.q.out
+++ /dev/null
@@ -1,65 +0,0 @@
-PREHOOK: query: select count(1) from src
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src
-#### A masked pattern was here ####
-GBY_2 ==> RS_3: 1
-TS_0 ==> SEL_1: 500
-SEL_1 ==> GBY_2: 500
-GBY_4 ==> SEL_5: 1
-main() ==> GBY_4: 1
-main() ==> TS_0: 500
-SEL_5 ==> FS_6: 1
-500
-PREHOOK: query: explain select count(1) from src
-PREHOOK: type: QUERY
-ABSTRACT SYNTAX TREE:
- (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION count 1)))))
-
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 is a root stage
-
-STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Alias -> Map Operator Tree:
- src
- TableScan
- alias: src
- Select Operator
- Group By Operator
- aggregations:
- expr: count(1)
- bucketGroup: false
- mode: hash
- outputColumnNames: _col0
- Reduce Output Operator
- sort order:
- tag: -1
- value expressions:
- expr: _col0
- type: bigint
- Reduce Operator Tree:
- Group By Operator
- aggregations:
- expr: count(VALUE._col0)
- bucketGroup: false
- mode: mergepartial
- outputColumnNames: _col0
- Select Operator
- expressions:
- expr: _col0
- type: bigint
- outputColumnNames: _col0
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
-
-
diff --git ql/src/test/results/clientpositive/hiveprofiler_script0.q.out ql/src/test/results/clientpositive/hiveprofiler_script0.q.out
deleted file mode 100644
index cc03f5d..0000000
--- ql/src/test/results/clientpositive/hiveprofiler_script0.q.out
+++ /dev/null
@@ -1,18 +0,0 @@
-PREHOOK: query: -- checking that script operator does not cause NPE
--- Derby strangeness is causing the output collector for the Hive Profiler to not get output during DB read
-
-SELECT TRANSFORM(src.key, src.value)
- USING 'testgrep' AS (tkey, tvalue)
-FROM src
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src
-#### A masked pattern was here ####
-103 val_103
-100 val_100
-10 val_10
-103 val_103
-104 val_104
-105 val_105
-104 val_104
-310 val_310
-100 val_100
diff --git ql/src/test/results/clientpositive/hiveprofiler_union0.q.out ql/src/test/results/clientpositive/hiveprofiler_union0.q.out
deleted file mode 100644
index ac5c9dc..0000000
--- ql/src/test/results/clientpositive/hiveprofiler_union0.q.out
+++ /dev/null
@@ -1,598 +0,0 @@
-PREHOOK: query: FROM (
- FROM src select src.key, src.value WHERE src.key < 100
- UNION ALL
- FROM src SELECT src.* WHERE src.key > 100
-) unioninput
-SELECT unioninput.*
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src
-#### A masked pattern was here ####
-SEL_5 ==> UNION_6: 414
-FIL_10 ==> SEL_5: 414
-TS_0 ==> FIL_9: 500
-TS_3 ==> FIL_10: 500
-UNION_6 ==> SEL_7: 498
-SEL_2 ==> UNION_6: 84
-FIL_9 ==> SEL_2: 84
-main() ==> TS_3: 500
-main() ==> TS_0: 500
-SEL_7 ==> FS_8: 498
-238 val_238
-86 val_86
-311 val_311
-27 val_27
-165 val_165
-409 val_409
-255 val_255
-278 val_278
-98 val_98
-484 val_484
-265 val_265
-193 val_193
-401 val_401
-150 val_150
-273 val_273
-224 val_224
-369 val_369
-66 val_66
-128 val_128
-213 val_213
-146 val_146
-406 val_406
-429 val_429
-374 val_374
-152 val_152
-469 val_469
-145 val_145
-495 val_495
-37 val_37
-327 val_327
-281 val_281
-277 val_277
-209 val_209
-15 val_15
-82 val_82
-403 val_403
-166 val_166
-417 val_417
-430 val_430
-252 val_252
-292 val_292
-219 val_219
-287 val_287
-153 val_153
-193 val_193
-338 val_338
-446 val_446
-459 val_459
-394 val_394
-237 val_237
-482 val_482
-174 val_174
-413 val_413
-494 val_494
-207 val_207
-199 val_199
-466 val_466
-208 val_208
-174 val_174
-399 val_399
-396 val_396
-247 val_247
-417 val_417
-489 val_489
-162 val_162
-377 val_377
-397 val_397
-309 val_309
-365 val_365
-266 val_266
-439 val_439
-342 val_342
-367 val_367
-325 val_325
-167 val_167
-195 val_195
-475 val_475
-17 val_17
-113 val_113
-155 val_155
-203 val_203
-339 val_339
-0 val_0
-455 val_455
-128 val_128
-311 val_311
-316 val_316
-57 val_57
-302 val_302
-205 val_205
-149 val_149
-438 val_438
-345 val_345
-129 val_129
-170 val_170
-20 val_20
-489 val_489
-157 val_157
-378 val_378
-221 val_221
-92 val_92
-111 val_111
-47 val_47
-72 val_72
-4 val_4
-280 val_280
-35 val_35
-427 val_427
-277 val_277
-208 val_208
-356 val_356
-399 val_399
-169 val_169
-382 val_382
-498 val_498
-125 val_125
-386 val_386
-437 val_437
-469 val_469
-192 val_192
-286 val_286
-187 val_187
-176 val_176
-54 val_54
-459 val_459
-51 val_51
-138 val_138
-103 val_103
-239 val_239
-213 val_213
-216 val_216
-430 val_430
-278 val_278
-176 val_176
-289 val_289
-221 val_221
-65 val_65
-318 val_318
-332 val_332
-311 val_311
-275 val_275
-137 val_137
-241 val_241
-83 val_83
-333 val_333
-180 val_180
-284 val_284
-12 val_12
-230 val_230
-181 val_181
-67 val_67
-260 val_260
-404 val_404
-384 val_384
-489 val_489
-353 val_353
-373 val_373
-272 val_272
-138 val_138
-217 val_217
-84 val_84
-348 val_348
-466 val_466
-58 val_58
-8 val_8
-411 val_411
-230 val_230
-208 val_208
-348 val_348
-24 val_24
-463 val_463
-431 val_431
-179 val_179
-172 val_172
-42 val_42
-129 val_129
-158 val_158
-119 val_119
-496 val_496
-0 val_0
-322 val_322
-197 val_197
-468 val_468
-393 val_393
-454 val_454
-298 val_298
-199 val_199
-191 val_191
-418 val_418
-96 val_96
-26 val_26
-165 val_165
-327 val_327
-230 val_230
-205 val_205
-120 val_120
-131 val_131
-51 val_51
-404 val_404
-43 val_43
-436 val_436
-156 val_156
-469 val_469
-468 val_468
-308 val_308
-95 val_95
-196 val_196
-288 val_288
-481 val_481
-457 val_457
-98 val_98
-282 val_282
-197 val_197
-187 val_187
-318 val_318
-318 val_318
-409 val_409
-470 val_470
-137 val_137
-369 val_369
-316 val_316
-169 val_169
-413 val_413
-85 val_85
-77 val_77
-0 val_0
-490 val_490
-87 val_87
-364 val_364
-179 val_179
-118 val_118
-134 val_134
-395 val_395
-282 val_282
-138 val_138
-238 val_238
-419 val_419
-15 val_15
-118 val_118
-72 val_72
-90 val_90
-307 val_307
-19 val_19
-435 val_435
-10 val_10
-277 val_277
-273 val_273
-306 val_306
-224 val_224
-309 val_309
-389 val_389
-327 val_327
-242 val_242
-369 val_369
-392 val_392
-272 val_272
-331 val_331
-401 val_401
-242 val_242
-452 val_452
-177 val_177
-226 val_226
-5 val_5
-497 val_497
-402 val_402
-396 val_396
-317 val_317
-395 val_395
-58 val_58
-35 val_35
-336 val_336
-95 val_95
-11 val_11
-168 val_168
-34 val_34
-229 val_229
-233 val_233
-143 val_143
-472 val_472
-322 val_322
-498 val_498
-160 val_160
-195 val_195
-42 val_42
-321 val_321
-430 val_430
-119 val_119
-489 val_489
-458 val_458
-78 val_78
-76 val_76
-41 val_41
-223 val_223
-492 val_492
-149 val_149
-449 val_449
-218 val_218
-228 val_228
-138 val_138
-453 val_453
-30 val_30
-209 val_209
-64 val_64
-468 val_468
-76 val_76
-74 val_74
-342 val_342
-69 val_69
-230 val_230
-33 val_33
-368 val_368
-103 val_103
-296 val_296
-113 val_113
-216 val_216
-367 val_367
-344 val_344
-167 val_167
-274 val_274
-219 val_219
-239 val_239
-485 val_485
-116 val_116
-223 val_223
-256 val_256
-263 val_263
-70 val_70
-487 val_487
-480 val_480
-401 val_401
-288 val_288
-191 val_191
-5 val_5
-244 val_244
-438 val_438
-128 val_128
-467 val_467
-432 val_432
-202 val_202
-316 val_316
-229 val_229
-469 val_469
-463 val_463
-280 val_280
-2 val_2
-35 val_35
-283 val_283
-331 val_331
-235 val_235
-80 val_80
-44 val_44
-193 val_193
-321 val_321
-335 val_335
-104 val_104
-466 val_466
-366 val_366
-175 val_175
-403 val_403
-483 val_483
-53 val_53
-105 val_105
-257 val_257
-406 val_406
-409 val_409
-190 val_190
-406 val_406
-401 val_401
-114 val_114
-258 val_258
-90 val_90
-203 val_203
-262 val_262
-348 val_348
-424 val_424
-12 val_12
-396 val_396
-201 val_201
-217 val_217
-164 val_164
-431 val_431
-454 val_454
-478 val_478
-298 val_298
-125 val_125
-431 val_431
-164 val_164
-424 val_424
-187 val_187
-382 val_382
-5 val_5
-70 val_70
-397 val_397
-480 val_480
-291 val_291
-24 val_24
-351 val_351
-255 val_255
-104 val_104
-70 val_70
-163 val_163
-438 val_438
-119 val_119
-414 val_414
-200 val_200
-491 val_491
-237 val_237
-439 val_439
-360 val_360
-248 val_248
-479 val_479
-305 val_305
-417 val_417
-199 val_199
-444 val_444
-120 val_120
-429 val_429
-169 val_169
-443 val_443
-323 val_323
-325 val_325
-277 val_277
-230 val_230
-478 val_478
-178 val_178
-468 val_468
-310 val_310
-317 val_317
-333 val_333
-493 val_493
-460 val_460
-207 val_207
-249 val_249
-265 val_265
-480 val_480
-83 val_83
-136 val_136
-353 val_353
-172 val_172
-214 val_214
-462 val_462
-233 val_233
-406 val_406
-133 val_133
-175 val_175
-189 val_189
-454 val_454
-375 val_375
-401 val_401
-421 val_421
-407 val_407
-384 val_384
-256 val_256
-26 val_26
-134 val_134
-67 val_67
-384 val_384
-379 val_379
-18 val_18
-462 val_462
-492 val_492
-298 val_298
-9 val_9
-341 val_341
-498 val_498
-146 val_146
-458 val_458
-362 val_362
-186 val_186
-285 val_285
-348 val_348
-167 val_167
-18 val_18
-273 val_273
-183 val_183
-281 val_281
-344 val_344
-97 val_97
-469 val_469
-315 val_315
-84 val_84
-28 val_28
-37 val_37
-448 val_448
-152 val_152
-348 val_348
-307 val_307
-194 val_194
-414 val_414
-477 val_477
-222 val_222
-126 val_126
-90 val_90
-169 val_169
-403 val_403
-400 val_400
-200 val_200
-97 val_97
-PREHOOK: query: explain
- FROM (
- FROM src select src.key, src.value WHERE src.key < 100
- UNION ALL
- FROM src SELECT src.* WHERE src.key > 100
- ) unioninput
- SELECT unioninput.*
-PREHOOK: type: QUERY
-ABSTRACT SYNTAX TREE:
- (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) value))) (TOK_WHERE (< (. (TOK_TABLE_OR_COL src) key) 100)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME src)))) (TOK_WHERE (> (. (TOK_TABLE_OR_COL src) key) 100))))) unioninput)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME unioninput))))))
-
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 is a root stage
-
-STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Alias -> Map Operator Tree:
- null-subquery1:unioninput-subquery1:src
- TableScan
- alias: src
- Filter Operator
- predicate:
- expr: (key < 100.0)
- type: boolean
- Select Operator
- expressions:
- expr: key
- type: string
- expr: value
- type: string
- outputColumnNames: _col0, _col1
- Union
- Select Operator
- expressions:
- expr: _col0
- type: string
- expr: _col1
- type: string
- outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- null-subquery2:unioninput-subquery2:src
- TableScan
- alias: src
- Filter Operator
- predicate:
- expr: (key > 100.0)
- type: boolean
- Select Operator
- expressions:
- expr: key
- type: string
- expr: value
- type: string
- outputColumnNames: _col0, _col1
- Union
- Select Operator
- expressions:
- expr: _col0
- type: string
- expr: _col1
- type: string
- outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
-
-
diff --git ql/src/test/results/clientpositive/operatorhook.q.out ql/src/test/results/clientpositive/operatorhook.q.out
deleted file mode 100644
index 541da0a..0000000
--- ql/src/test/results/clientpositive/operatorhook.q.out
+++ /dev/null
@@ -1,7 +0,0 @@
-PREHOOK: query: SELECT count(1) FROM src
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src
-#### A masked pattern was here ####
-TEST_OPERATOR_HOOK_EXIT: 7508
-TEST_OPERATOR_HOOK_ENTER: 7508
-500