diff --git build-common.xml build-common.xml index 96e8046..5b7514b 100644 --- build-common.xml +++ build-common.xml @@ -59,7 +59,7 @@ - + diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 78d9cc9..187d2fa 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -190,7 +190,6 @@ PREEXECHOOKS("hive.exec.pre.hooks", ""), POSTEXECHOOKS("hive.exec.post.hooks", ""), ONFAILUREHOOKS("hive.exec.failure.hooks", ""), - OPERATORHOOKS("hive.exec.operator.hooks", ""), CLIENTSTATSPUBLISHERS("hive.client.stats.publishers", ""), EXECPARALLEL("hive.exec.parallel", false), // parallel query launching EXECPARALLETHREADNUMBER("hive.exec.parallel.thread.number", 8), @@ -565,18 +564,6 @@ HIVE_INDEX_COMPACT_QUERY_MAX_SIZE("hive.index.compact.query.max.size", (long) 10 * 1024 * 1024 * 1024), // 10G HIVE_INDEX_COMPACT_BINARY_SEARCH("hive.index.compact.binary.search", true), - //Profiler - HIVEPROFILERDBCLASS("hive.profiler.dbclass","jdbc:derby"), - HIVEPROFILERJDBCDRIVER("hive.profiler.jdbcdriver", "org.apache.derby.jdbc.EmbeddedDriver"), - HIVEPROFILERDBCONNECTIONSTRING("hive.profiler.dbconnectionstring", - "jdbc:derby:;databaseName=TempProfilerStore;create=true"), // automatically create database - // default timeout for JDBC connection - HIVE_PROFILER_JDBC_TIMEOUT("hive.profiler.jdbc.timeout", 30), - HIVE_PROFILER_RETRIES_MAX("hive.stats.retries.max", - 0), // maximum # of retries to insert/select/delete the stats DB - HIVE_PROFILER_RETRIES_WAIT("hive.stats.retries.wait", - 3000), // # milliseconds to wait before the next retry - // Statistics HIVESTATSAUTOGATHER("hive.stats.autogather", true), HIVESTATSDBCLASS("hive.stats.dbclass", diff --git conf/hive-default.xml.template conf/hive-default.xml.template index e266ce7..02e1cca 100644 --- conf/hive-default.xml.template +++ conf/hive-default.xml.template @@ -1123,42 +1123,6 @@ - hive.profiler.dbclass - jdbc:derby - The default database that stores temporary hive statistics. - - - - hive.profiler.jdbcdriver - org.apache.derby.jdbc.EmbeddedDriver - The JDBC driver for the database that stores temporary hive statistics. - - - - hive.profiler.dbconnectionstring - jdbc:derby:;databaseName=TempStatsStore;create=true - The default connection string for the database that stores temporary hive statistics. - - - - hive.profiler.jdbc.timeout - 30 - Timeout value (number of seconds) used by JDBC connection and statements. - - - - hive.profiler.retries.max - 0 - Maximum number of retries when profiler publisher/aggregator got an exception updating intermediate database. Default is no tries on failures. - - - - hive.profiler.retries.wait - 3000 - The base waiting window (in milliseconds) before the next retry. The actual wait time is calculated by baseWindow * failues baseWindow * (failure 1) * (random number between [0.0,1.0]). - - - hive.stats.dbclass jdbc:derby The default database that stores temporary hive statistics. diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java index e5bbabf..08ffcb1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java @@ -50,7 +50,6 @@ private JobConf jc; private boolean abort = false; private Reporter rp; - private List opHooks; public static final Log l4j = LogFactory.getLog("ExecMapper"); private static boolean done; @@ -99,7 +98,6 @@ public void configure(JobConf job) { mo.setExecContext(execContext); mo.initializeLocalWork(jc); mo.initialize(jc, null); - opHooks = OperatorHookUtils.getOperatorHooks(jc); if (localWork == null) { return; @@ -132,7 +130,6 @@ public void map(Object key, Object value, OutputCollector output, rp = reporter; mo.setOutputCollector(oc); mo.setReporter(rp); - mo.setOperatorHooks(opHooks); MapredContext.get().setReporter(reporter); } // reset the execContext for each new row diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java index a5fce39..89d9643 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java @@ -66,7 +66,6 @@ private long nextCntr = 1; private static String[] fieldNames; - private List opHooks; public static final Log l4j = LogFactory.getLog("ExecReducer"); private boolean isLogInfoEnabled = false; @@ -152,7 +151,6 @@ public void configure(JobConf job) { try { l4j.info(reducer.dump(0)); reducer.initialize(jc, rowObjectInspector); - opHooks = OperatorHookUtils.getOperatorHooks(jc); } catch (Throwable e) { abort = true; if (e instanceof OutOfMemoryError) { @@ -183,7 +181,6 @@ public void reduce(Object key, Iterator values, OutputCollector output, rp = reporter; reducer.setOutputCollector(oc); reducer.setReporter(rp); - reducer.setOperatorHooks(opHooks); MapredContext.get().setReporter(reporter); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java index 0856ce9..6efa294 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java @@ -754,7 +754,9 @@ private FSPaths getDynOutPaths(List row, String lbDirName) throws HiveEx // check # of dp if (valToPaths.size() > maxPartitions) { // throw fatal error - incrCounter(fatalErrorCntr, 1); + if (counterNameToEnum != null) { + incrCounter(fatalErrorCntr, 1); + } fatalError = true; LOG.error("Fatal error was thrown due to exceeding number of dynamic partitions"); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java index 90e1333..e9e968c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java @@ -1076,7 +1076,7 @@ public void closeOp(boolean abort) throws HiveException { try { // put the hash related stats in statsMap if applicable, so that they // are sent to jt as counters - if (hashAggr) { + if (hashAggr && counterNameToEnum != null) { incrCounter(counterNameHashOut, numRowsHashTbl); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java index daf1894..6a54cbf 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java @@ -55,7 +55,6 @@ // Bean methods private static final long serialVersionUID = 1L; - List operatorHooks; private Configuration configuration; protected List> childOperators; @@ -241,17 +240,6 @@ public String getIdentifier() { return id; } - public void setOperatorHooks(List opHooks){ - operatorHooks = opHooks; - if (childOperators == null) { - return; - } - - for (Operator op : childOperators) { - op.setOperatorHooks(opHooks); - } - } - public void setReporter(Reporter rep) { reporter = rep; @@ -436,34 +424,6 @@ public void passExecContext(ExecMapperContext execContext) { } } - private void enterOperatorHooks(OperatorHookContext opHookContext) throws HiveException { - if (this.operatorHooks == null) { - return; - } - for(OperatorHook opHook : this.operatorHooks) { - opHook.enter(opHookContext); - } - } - - private void exitOperatorHooks(OperatorHookContext opHookContext) throws HiveException { - if (this.operatorHooks == null) { - return; - } - for(OperatorHook opHook : this.operatorHooks) { - opHook.exit(opHookContext); - } - } - - private void closeOperatorHooks(OperatorHookContext opHookContext) throws HiveException { - if (this.operatorHooks == null) { - return; - } - for(OperatorHook opHook : this.operatorHooks) { - opHook.close(opHookContext); - } - } - - /** * Collects all the parent's output object inspectors and calls actual * initialization method. @@ -525,12 +485,22 @@ public void process(Object row, int tag) throws HiveException { if (fatalError) { return; } - OperatorHookContext opHookContext = new OperatorHookContext(this, row, tag); - preProcessCounter(); - enterOperatorHooks(opHookContext); - processOp(row, tag); - exitOperatorHooks(opHookContext); - postProcessCounter(); + + if (counterNameToEnum != null) { + inputRows++; + if ((inputRows % 1000) == 0) { + incrCounter(numInputRowsCntr, inputRows); + incrCounter(timeTakenCntr, totalTime); + inputRows = 0; + totalTime = 0; + } + + beginTime = System.currentTimeMillis(); + processOp(row, tag); + totalTime += (System.currentTimeMillis() - beginTime); + } else { + processOp(row, tag); + } } // If a operator wants to do some work at the beginning of a group @@ -606,13 +576,14 @@ public void close(boolean abort) throws HiveException { state = State.CLOSE; LOG.info(id + " finished. closing... "); - incrCounter(numInputRowsCntr, inputRows); - incrCounter(numOutputRowsCntr, outputRows); - incrCounter(timeTakenCntr, totalTime); + if (counterNameToEnum != null) { + incrCounter(numInputRowsCntr, inputRows); + incrCounter(numOutputRowsCntr, outputRows); + incrCounter(timeTakenCntr, totalTime); + } LOG.info(id + " forwarded " + cntr + " rows"); - closeOperatorHooks(new OperatorHookContext(this)); // call the operator specific close routine closeOp(abort); @@ -822,9 +793,11 @@ private long getNextCntr(long cntr) { protected void forward(Object row, ObjectInspector rowInspector) throws HiveException { - if ((++outputRows % 1000) == 0) { - incrCounter(numOutputRowsCntr, outputRows); - outputRows = 0; + if (counterNameToEnum != null) { + if ((++outputRows % 1000) == 0) { + incrCounter(numOutputRowsCntr, outputRows); + outputRows = 0; + } } if (isLogInfoEnabled) { @@ -1158,39 +1131,12 @@ protected static StructObjectInspector initEvaluatorsAndReturnStruct( protected transient Object groupKeyObject; /** - * this is called before operator process to buffer some counters. - */ - private void preProcessCounter() { - inputRows++; - if ((inputRows % 1000) == 0) { - incrCounter(numInputRowsCntr, inputRows); - incrCounter(timeTakenCntr, totalTime); - inputRows = 0; - totalTime = 0; - } - beginTime = System.currentTimeMillis(); - } - - /** - * this is called after operator process to buffer some counters. - */ - private void postProcessCounter() { - if (counterNameToEnum != null) { - totalTime += (System.currentTimeMillis() - beginTime); - } - } - - /** * this is called in operators in map or reduce tasks. * * @param name * @param amount */ protected void incrCounter(String name, long amount) { - if(counterNameToEnum == null) { - return; - } - String counterName = getWrappedCounterName(name); ProgressCounter pc = counterNameToEnum.get(counterName); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorHook.java ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorHook.java deleted file mode 100644 index a3d49d1..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorHook.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec; - -import org.apache.hadoop.hive.ql.metadata.HiveException; -/** - * OperatorHook is a new interface which - * executes pre/post each Operator execution in a map/reduce stage - */ - -public interface OperatorHook { - public void enter(OperatorHookContext opHookContext) throws HiveException; - public void exit(OperatorHookContext opHookContext) throws HiveException; - public void close(OperatorHookContext opHookContext) throws HiveException; -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorHookContext.java ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorHookContext.java deleted file mode 100644 index a27a653..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorHookContext.java +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec; - -import java.util.List; - -public class OperatorHookContext { - private final String operatorName; - private final String operatorId; - private final Object currentRow; - private final int parentTag; - private Operator operator; - - public OperatorHookContext(Operator op) { - this(op, null, -1); - } - - public OperatorHookContext(Operator op, Object row, int tag) { - this(op.getName(), op.getIdentifier(), row, tag); - this.operator = op; - } - - private OperatorHookContext(String opName, String opId, Object row, int tag) { - operatorName = opName; - operatorId = opId; - currentRow = row; - parentTag = tag; - } - - public Operator getOperator() { - return operator; - } - - public Operator getParentOperator() { - List parents = this.operator.getParentOperators(); - if (parents == null || parents.isEmpty()) { - return null; - } - return (Operator)(this.operator.getParentOperators().get(this.parentTag)); - - } - - public String getOperatorName() { - return operatorName; - } - - public String getOperatorId() { - return operatorId; - } - - public Object getCurrentRow() { - return currentRow; - } - - @Override - public String toString() { - return "operatorName= " + this.getOperatorName() + - ", id=" + this.getOperatorId(); - - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorHookUtils.java ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorHookUtils.java deleted file mode 100644 index d7be357..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorHookUtils.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec; -import java.util.List; -import java.util.ArrayList; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.common.JavaUtils; -import org.apache.hadoop.hive.conf.HiveConf; -public class OperatorHookUtils { - - public static final Log LOG = LogFactory.getLog("OperatorHookUtils"); - public static List getOperatorHooks(Configuration hconf) - throws Exception { - List opHooks = new ArrayList(); - String csOpHooks = HiveConf.getVar(hconf, HiveConf.ConfVars.OPERATORHOOKS); - if(csOpHooks == null) { - return opHooks; - } - String[] opHookClasses = csOpHooks.split(","); - for(String opHookClass: opHookClasses) { - try { - OperatorHook opHook = - (OperatorHook) Class.forName(opHookClass.trim(), true, - JavaUtils.getClassLoader()).newInstance(); - opHooks.add(opHook); - } catch (ClassNotFoundException e) { - LOG.error(opHookClass + " Class not found: " + e.getMessage()); - } - } - return opHooks; - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilePublisher.java ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilePublisher.java deleted file mode 100644 index 6c46bac..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilePublisher.java +++ /dev/null @@ -1,111 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.profiler; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.util.Map; -import java.sql.SQLException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.exec.Utilities; - -public class HiveProfilePublisher { - private final Log LOG = LogFactory.getLog(this.getClass().getName()); - private boolean connected = false; - private HiveProfilePublisherInfo info; - - - public boolean closeConnection() { - - if (info == null || info.getConnection() == null) { - return true; - } - try { - if (info.getInsertStatement() != null) { - info.closeInsertStatement(); - } - info.getConnection().close(); - return true; - } catch (Exception e) { - LOG.error("Error during JDBC termination. ", e); - return false; - } - } - - public boolean initialize(Configuration conf) { - try { - info = new HiveProfilePublisherInfo(conf); - String createTable = getCreate(); - LOG.info(createTable); - HiveProfilerUtils.createTableIfNonExistent(info, createTable); - info.prepareInsert(); - } catch (Exception e) { - LOG.error("Error during HiveProfilePublisher initialization", e); - return false; - } - return true; - } - - private String getCreate() { - return "CREATE TABLE " + info.getTableName() + - " ( " + - HiveProfilerStats.Columns.QUERY_ID + " VARCHAR(512) NOT NULL, " + - HiveProfilerStats.Columns.TASK_ID + " VARCHAR(512) NOT NULL, " + - HiveProfilerStats.Columns.OPERATOR_ID + " INT, " + - HiveProfilerStats.Columns.OPERATOR_NAME + " VARCHAR(512) NOT NULL, " + - HiveProfilerStats.Columns.PARENT_OPERATOR_ID + " INT," + - HiveProfilerStats.Columns.PARENT_OPERATOR_NAME + " VARCHAR(512), " + - HiveProfilerStats.Columns.LEVEL_ANNO_NAME + " VARCHAR(512), " + - HiveProfilerStats.Columns.CALL_COUNT + " BIGINT, " + - HiveProfilerStats.Columns.INCL_TIME + " BIGINT ) "; - - } - - public boolean publishStat(String queryId, Map stats, - Configuration conf) { - if (info == null || info.getConnection() == null) { - if(!initialize(conf)) { - return false; - } - } else { - try { - if(stats == null || stats.isEmpty()) { - return true; - } - Utilities.SQLCommand execUpdate = new Utilities.SQLCommand() { - @Override - public Void run(PreparedStatement stmt) throws SQLException { - stmt.executeUpdate(); - return null; - } - }; - PreparedStatement insStmt = info.getInsert(stats); - Utilities.executeWithRetry(execUpdate, insStmt, info.getWaitWindow(), info.getMaxRetries()); - } catch (Exception e) { - LOG.error("ERROR during publishing profiling data. ", e); - return false; - } - } - - return true; - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilePublisherInfo.java ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilePublisherInfo.java deleted file mode 100644 index 0b6baa0..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilePublisherInfo.java +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.profiler; - -import java.util.Map; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.sql.Connection; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.exec.Utilities; - -public class HiveProfilePublisherInfo implements HiveProfilerConnectionInfo { - final private Log LOG = LogFactory.getLog(this.getClass().getName()); - private String statsDbClass; - private final String tableName = "PROFILER_STATS"; - private int maxRetries, waitWindow, timeout; - private Connection conn; - private String connectionString; - private PreparedStatement insStmt; - - public String getDbClass() { return statsDbClass; } - - public int getTimeout() { return timeout; } - - public int getMaxRetries() { return maxRetries; } - - public int getWaitWindow() { return waitWindow; } - - public String getConnectionString() { return connectionString; } - - public String getTableName() { return tableName; } - - public Connection getConnection() { return conn; } - - protected PreparedStatement getInsertStatement() { - return insStmt; - } - - private String getInsert() { - String colNames = ""; - String val = ""; - int numCols = HiveProfilerStats.COLUMN_NAMES.length; - for (int i = 0; i < numCols; i++) { - colNames += HiveProfilerStats.COLUMN_NAMES[i]; - val += "?"; - - if (i < numCols - 1) { - colNames += ","; - val += ","; - } - } - return "INSERT INTO " + tableName + " (" + colNames + ") VALUES (" + val + ")"; - } - - public HiveProfilePublisherInfo (Configuration conf) throws Exception{ - maxRetries = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_PROFILER_RETRIES_MAX); - waitWindow = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_PROFILER_RETRIES_WAIT); - connectionString = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEPROFILERDBCONNECTIONSTRING); - timeout = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_PROFILER_JDBC_TIMEOUT); - String driver = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEPROFILERJDBCDRIVER); - statsDbClass = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEPROFILERDBCLASS); - try { - Class.forName(driver).newInstance(); - } catch (Exception e) { - LOG.error("Error during instantiating JDBC driver " + driver + ". ", e); - } - DriverManager.setLoginTimeout(timeout); // stats is non-blocking - conn = Utilities.connectWithRetry(connectionString, waitWindow, maxRetries); - } - - protected void prepareInsert() throws SQLException { - insStmt = Utilities.prepareWithRetry(conn, getInsert(), waitWindow, maxRetries); - } - protected void closeInsertStatement() throws SQLException { - insStmt.close(); - } - - protected PreparedStatement getInsert(Map stats) throws SQLException { - for (int i = 0; i < HiveProfilerStats.COLUMN_NAMES.length; i++) { - insStmt.setString(i + 1, stats.get(HiveProfilerStats.COLUMN_NAMES[i])); - } - return insStmt; - } - -} - diff --git ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfiler.java ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfiler.java deleted file mode 100644 index bdc9e50..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfiler.java +++ /dev/null @@ -1,127 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.profiler; - -import java.util.Collection; -import java.util.Iterator; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.ql.exec.OperatorHook; -import org.apache.hadoop.hive.ql.exec.OperatorHookContext; -import org.apache.hadoop.hive.ql.metadata.HiveException; - -public class HiveProfiler implements OperatorHook { - private final Log LOG = LogFactory.getLog(this.getClass().getName()); - private static final HiveProfilePublisher pub = new HiveProfilePublisher(); - - private final Map operatorCallStack = - new ConcurrentHashMap(); - - // Aggregates stats for each operator in memory so that stats are written to DB - // all at once - this allows the profiler to be extremely lightweight in - // communication with the DB - private final Map aggrStats = - new ConcurrentHashMap(); - - public void enter(OperatorHookContext opHookContext) throws HiveException { - String opLevelAnnoName = HiveProfilerUtils.getLevelAnnotatedName(opHookContext); - HiveProfilerEntry curEntry = new HiveProfilerEntry(opHookContext); - assert(operatorCallStack.get(opLevelAnnoName) == null); - operatorCallStack.put(opLevelAnnoName, curEntry); - } - - private void exit(HiveProfilerEntry curEntry) { - OperatorHookContext opHookContext = curEntry.getOperatorHookContext(); - // update the metrics we are - long exitTime = System.nanoTime(); - long wallTime = exitTime - curEntry.wallStartTime; - - String opName = opHookContext.getOperatorName(); - - Configuration conf = opHookContext.getOperator().getConfiguration(); - - String opLevelAnnoName = HiveProfilerUtils.getLevelAnnotatedName(opHookContext); - - if (aggrStats.containsKey(opLevelAnnoName)) { - aggrStats.get(opLevelAnnoName).updateStats(wallTime, 1); - } else { - HiveProfilerStats stats = - new HiveProfilerStats(opHookContext, 1, wallTime, conf); - aggrStats.put(opLevelAnnoName, stats); - } - - } - public void exit(OperatorHookContext opHookContext) throws HiveException { - if (operatorCallStack.isEmpty()) { - LOG.error("Unexpected state: Operator Call Stack is empty on exit."); - } - String opLevelAnnoName = HiveProfilerUtils.getLevelAnnotatedName(opHookContext); - - HiveProfilerEntry curEntry = operatorCallStack.get(opLevelAnnoName); - - if (!curEntry.getOperatorHookContext().equals(opHookContext)) { - LOG.error("Expected to exit from: " + curEntry.getOperatorHookContext().toString() + - " but exit called on " + opHookContext.toString()); - } - - exit(curEntry); - operatorCallStack.remove(opLevelAnnoName); - } - - public void close(OperatorHookContext opHookContext) { - Configuration conf = opHookContext.getOperator().getConfiguration(); - - Collection stats = aggrStats.values(); - // example: - // queryId=pamelavagata_20130115163838_4a1cb4ae-43c1-4656-bfae-118557896eec, - // operatorName=TS, - // id=3, - // parentName="" (root), - // inclTime=1202710 - // callCount - - Iterator statsIter = stats.iterator(); - while (statsIter.hasNext()) { - HiveProfilerStats stat = statsIter.next(); - pub.initialize(conf); - boolean published = pub.publishStat(null, stat.getStatsMap(), conf); - LOG.info((published ? "did " : "did not ") + "publish stat for: " + stat.toString()); - pub.closeConnection(); - } - stats.clear(); - - } - - private class HiveProfilerEntry { - OperatorHookContext ctxt; - protected long wallStartTime; - - protected HiveProfilerEntry(OperatorHookContext opHookContext) { - this.ctxt = opHookContext; - this.wallStartTime = System.nanoTime(); - } - - protected OperatorHookContext getOperatorHookContext() { - return ctxt; - } - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerAggregateStat.java ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerAggregateStat.java deleted file mode 100644 index 151ea4a..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerAggregateStat.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.profiler; - -public class HiveProfilerAggregateStat { - long wallTime; - long callCount; - public HiveProfilerAggregateStat(long wallTime, long callCount){ - this.wallTime = wallTime; - this.callCount = callCount; - } - - public long getCallCount() { - return callCount; - } - - public long getWallTime() { - return wallTime; - } - - /* - * @param wallTime: inclusive walltime in microseconds - */ - public void update(long wallTime, long callCount) { - this.wallTime += wallTime; - this.callCount += callCount; - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerConnectionInfo.java ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerConnectionInfo.java deleted file mode 100644 index cd8f46d..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerConnectionInfo.java +++ /dev/null @@ -1,29 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -package org.apache.hadoop.hive.ql.profiler; - -import java.sql.Connection; -public interface HiveProfilerConnectionInfo { - public String getDbClass(); - public int getTimeout(); - public int getMaxRetries(); - public int getWaitWindow(); - public String getConnectionString(); - public String getTableName(); - public Connection getConnection(); -} diff --git ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerStats.java ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerStats.java deleted file mode 100644 index e4ee89c..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerStats.java +++ /dev/null @@ -1,108 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.profiler; -import java.util.HashMap; -import java.util.Map; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.OperatorHookContext; -import org.apache.hadoop.hive.ql.exec.Utilities; - -public class HiveProfilerStats { - public final class Columns { - public static final String QUERY_ID = "queryId"; - public static final String OPERATOR_NAME = "operatorName"; - public static final String OPERATOR_ID = "operatorId"; - public static final String PARENT_OPERATOR_ID = "parentOperatorId"; - public static final String PARENT_OPERATOR_NAME = "parentOperatorName"; - public static final String EXCL_TIME = "exclTime"; - public static final String LEVEL_ANNO_NAME = "levelAnnotatedName"; - public static final String INCL_TIME = "inclTime"; - public static final String CALL_COUNT = "callCount"; - public static final String TASK_ID = "taskId"; - } - - public static final String[] COLUMN_NAMES= new String[] { - Columns.QUERY_ID, - Columns.TASK_ID, - Columns.OPERATOR_NAME, - Columns.OPERATOR_ID, - Columns.PARENT_OPERATOR_ID, - Columns.PARENT_OPERATOR_NAME, - Columns.LEVEL_ANNO_NAME, - Columns.INCL_TIME, - Columns.CALL_COUNT - }; - - private final Map stats = new HashMap(); - - long callCount; - long inclTime; - String taskId; - - protected HiveProfilerStats( - OperatorHookContext opHookContext, - long callCount, long wallTime, Configuration conf) { - this.callCount = callCount; - this.inclTime = wallTime; - this.taskId = Utilities.getTaskId(conf); - populateStatsMap(opHookContext, conf); - } - - private void populateStatsMap(OperatorHookContext opHookContext, - Configuration conf) { - String queryId = - conf == null ? "no conf" : HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYID); - stats.put(Columns.QUERY_ID, queryId); - String opName = opHookContext.getOperatorName(); - stats.put( - Columns.OPERATOR_NAME, opName); - stats.put( - Columns.OPERATOR_ID, opHookContext.getOperatorId()); - - Operator parent = opHookContext.getParentOperator(); - String parentOpName = parent == null ? "" : parent.getName(); - stats.put(Columns.PARENT_OPERATOR_NAME, parentOpName); - - String parentOpId = parent == null ? "-1" : parent.getIdentifier(); - stats.put(Columns.PARENT_OPERATOR_ID, parentOpId); - - stats.put(Columns.LEVEL_ANNO_NAME, HiveProfilerUtils.getLevelAnnotatedName(opHookContext)); - - } - - public void updateStats(long wallTime, long count) { - this.inclTime += wallTime; - this.callCount += count; - } - - public Map getStatsMap() { - stats.put(Columns.TASK_ID, taskId); - stats.put(Columns.INCL_TIME, String.valueOf(inclTime)); - stats.put(Columns.CALL_COUNT, String.valueOf(callCount)); - return stats; - } - - @Override - public String toString() { - return stats.toString(); - } -} - diff --git ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerStatsAggregator.java ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerStatsAggregator.java deleted file mode 100644 index d640056..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerStatsAggregator.java +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.profiler; - -import java.util.HashMap; -import java.util.Map; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.PreparedStatement; -import java.sql.SQLException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.exec.Utilities; - -public class HiveProfilerStatsAggregator { - final private Log LOG = LogFactory.getLog(this.getClass().getName()); - private long totalTime; - private HiveProfilePublisherInfo rawProfileConnInfo; - - private Map stats = - new HashMap(); - - public HiveProfilerStatsAggregator(HiveConf conf) { - try { - // initialize the raw data connection - rawProfileConnInfo = new HiveProfilePublisherInfo(conf); - populateAggregateStats(conf); - } catch (Exception e) { - - LOG.error("Error during initialization", e); - } - } - - public long getTotalTime() { - return totalTime; - } - - public Map getAggregateStats() { - return stats; - } - - - private void populateAggregateStats(HiveConf conf) { - int waitWindow = rawProfileConnInfo.getWaitWindow(); - int maxRetries = rawProfileConnInfo.getMaxRetries(); - - String queryId = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYID); - String profilerStatsTable = rawProfileConnInfo.getTableName(); - String getProfileStats = - "SELECT * FROM " + profilerStatsTable + " WHERE queryId = ? "; - Utilities.SQLCommand execQuery = new Utilities.SQLCommand() { - @Override - public ResultSet run(PreparedStatement stmt) throws SQLException { - return stmt.executeQuery(); - } - }; - - try { - PreparedStatement getProfileStatsStmt = - Utilities.prepareWithRetry(rawProfileConnInfo.getConnection(), - getProfileStats, waitWindow, maxRetries); - getProfileStatsStmt.setString(1, queryId); - ResultSet result = Utilities.executeWithRetry(execQuery, getProfileStatsStmt, - waitWindow, maxRetries); - - populateAggregateStats(result); - getProfileStatsStmt.close(); - rawProfileConnInfo.getConnection().close(); - } catch(Exception e) { - LOG.error("executing error: ", e); - } - } - - private void populateAggregateStats(ResultSet result) { - try { - while(result.next()){ - // string denoting parent==>child - // example:SEL_2==>GBY_1 - String levelAnnoName = result.getString(HiveProfilerStats.Columns.LEVEL_ANNO_NAME); - // Microseconds - Long curInclTime = result.getLong(HiveProfilerStats.Columns.INCL_TIME) / 1000; - Long curCallCount = result.getLong(HiveProfilerStats.Columns.CALL_COUNT); - totalTime += curInclTime; - if(curInclTime != null && curCallCount != null) { - HiveProfilerAggregateStat curStat; - if (stats.containsKey(levelAnnoName)) { - curStat = stats.get(levelAnnoName); - curStat.update(curInclTime, curCallCount); - } else { - curStat = new HiveProfilerAggregateStat(curInclTime, curCallCount); - } - stats.put(levelAnnoName, curStat); - } - } - } catch (Exception e) { - LOG.error("Error Aggregating Stats", e); - } - } - -} - diff --git ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerUtils.java ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerUtils.java deleted file mode 100644 index d220ed7..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/profiler/HiveProfilerUtils.java +++ /dev/null @@ -1,73 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -package org.apache.hadoop.hive.ql.profiler; - -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.DriverManager; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; - -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.MapOperator; -import org.apache.hadoop.hive.ql.exec.OperatorHookContext; - -public class HiveProfilerUtils { - public static void createTableIfNonExistent(HiveProfilerConnectionInfo info, - String createTable) throws Exception { - Connection conn = info.getConnection(); - Statement stmt = conn.createStatement(); - stmt.setQueryTimeout(info.getTimeout()); - DatabaseMetaData dbm = conn.getMetaData(); - ResultSet rs = dbm.getTables(null, null, info.getTableName(), null); - boolean tblExists = rs.next(); - if(!tblExists) { - stmt.executeUpdate(createTable); - stmt.close(); - } - } - - public static boolean closeConnection(HiveProfilerConnectionInfo info) throws SQLException{ - info.getConnection().close(); - // In case of derby, explicitly shutdown the database otherwise it reports error when - // trying to connect to the same JDBC connection string again. - if (info.getDbClass().equalsIgnoreCase("jdbc:derby")) { - try { - // The following closes the derby connection. It throws an exception that has to be caught - // and ignored. - DriverManager.getConnection(info.getConnectionString() + ";shutdown=true"); - } catch (Exception e) { - // Do nothing because we know that an exception is thrown anyway. - } - } - return true; - } - - public static String getLevelAnnotatedName(OperatorHookContext opHookContext) { - Operator parent = opHookContext.getParentOperator(); - if (parent != null && parent instanceof MapOperator) { - parent = null; - } - Operator op = opHookContext.getOperator(); - String parentOpName = parent == null ? "" : parent.getName(); - String parentOpId = parent == null ? "main()" : parent.getOperatorId(); - String levelAnnoName = parentOpId + " ==> " + op.getOperatorId(); - return levelAnnoName; - } -} diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/TstOperatorHook.java ql/src/test/org/apache/hadoop/hive/ql/exec/TstOperatorHook.java deleted file mode 100644 index e2f318b..0000000 --- ql/src/test/org/apache/hadoop/hive/ql/exec/TstOperatorHook.java +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.exec; -import java.util.HashMap; - -import java.util.ArrayList; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -public class TstOperatorHook implements OperatorHook { - protected transient Log LOG = LogFactory.getLog(this.getClass().getName()); - private long enters=0; - private long exits=0; - public void enter(OperatorHookContext opHookContext) { - enters++; - } - - public void exit(OperatorHookContext opHookContext) { - exits++; - } - - public void close(OperatorHookContext opHookContext) { - incrCounter(TstOperatorHookUtils.TEST_OPERATOR_HOOK_ENTER, opHookContext, enters); - incrCounter(TstOperatorHookUtils.TEST_OPERATOR_HOOK_EXIT, opHookContext, exits); - } - - private void incrCounter(String ctrName, OperatorHookContext opHookContext, long incrVal) { - TstOperatorHookUtils.TestOperatorHookCounter ctr = - TstOperatorHookUtils.TestOperatorHookCounter.valueOf(ctrName); - Operator op = opHookContext.getOperator(); - LOG.info(ctrName); - op.reporter.incrCounter(ctr, incrVal); - Long val = op.reporter.getCounter(ctr).getValue(); - LOG.info(ctrName + " " + String.valueOf(val)); - } -} - diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/TstOperatorHookUtils.java ql/src/test/org/apache/hadoop/hive/ql/exec/TstOperatorHookUtils.java deleted file mode 100644 index 548336a..0000000 --- ql/src/test/org/apache/hadoop/hive/ql/exec/TstOperatorHookUtils.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec; - -import java.util.Collection; -import java.io.Serializable; - -import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; - -public class TstOperatorHookUtils { - public static final String TEST_OPERATOR_HOOK_ENTER = "TEST_OPERATOR_HOOK_ENTER"; - public static final String TEST_OPERATOR_HOOK_EXIT = "TEST_OPERATOR_HOOK_EXIT"; - - public static enum TestOperatorHookCounter { - TEST_OPERATOR_HOOK_ENTER, - TEST_OPERATOR_HOOK_EXIT, - } -} diff --git ql/src/test/org/apache/hadoop/hive/ql/hooks/HiveProfilerResultsHook.java ql/src/test/org/apache/hadoop/hive/ql/hooks/HiveProfilerResultsHook.java deleted file mode 100644 index a3df0cd..0000000 --- ql/src/test/org/apache/hadoop/hive/ql/hooks/HiveProfilerResultsHook.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.hooks; -import java.util.Map; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import java.sql.Connection; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.profiler.HiveProfilerStatsAggregator; -import org.apache.hadoop.hive.ql.profiler.HiveProfilerAggregateStat; -import org.apache.hadoop.hive.ql.session.SessionState; -import java.sql.DriverManager; -import org.apache.hadoop.hive.ql.exec.Utilities; - -public class HiveProfilerResultsHook implements ExecuteWithHookContext { - final private Log LOG = LogFactory.getLog(this.getClass().getName()); - @Override - public void run (HookContext hookContext) { - SessionState sess = SessionState.get(); - HiveConf conf = sess.getConf(); - HiveProfilerStatsAggregator aggr = new HiveProfilerStatsAggregator(conf); - Map stats = aggr.getAggregateStats(); - for (String name: stats.keySet()){ - SessionState.getConsole().printError(name + ": " + stats.get(name).getCallCount()); - } - } -} diff --git ql/src/test/org/apache/hadoop/hive/ql/hooks/PostTestOperatorHook.java ql/src/test/org/apache/hadoop/hive/ql/hooks/PostTestOperatorHook.java deleted file mode 100644 index b931209..0000000 --- ql/src/test/org/apache/hadoop/hive/ql/hooks/PostTestOperatorHook.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.hooks; - -import java.util.Collection; -import java.util.List; -import java.util.Iterator; -import java.io.Serializable; -import java.io.IOException; - -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.exec.Task; -import org.apache.hadoop.hive.ql.exec.TaskRunner; -import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; -import org.apache.hadoop.mapred.Counters; - -public class PostTestOperatorHook implements ExecuteWithHookContext { - private void logEnterExitCounters(Task task) throws IOException { - if(task.getTaskHandle() != null) { - Counters counters = task.getTaskHandle().getCounters(); - if(counters != null) { - logCounterValue(counters, "TEST_OPERATOR_HOOK_"); - } else { - SessionState.getConsole().printError("counters are null"); - } - } else { - SessionState.getConsole().printError("task handle is null"); - } - } - - private void logCounterValue(Counters ctr, String name) { - Collection counterGroups = ctr.getGroupNames(); - for (String groupName : counterGroups) { - Counters.Group group = ctr.getGroup(groupName); - Iterator it = group.iterator(); - while (it.hasNext()) { - Counters.Counter counter = it.next(); - if(counter.getName().contains(name)) { - SessionState.getConsole().printError(counter.getName() + ": " + counter.getValue()); - } - } - } - } - - public void run(HookContext hookContext) { - HiveConf conf = hookContext.getConf(); - List completedTasks = hookContext.getCompleteTaskList(); - if (completedTasks != null) { - for (TaskRunner taskRunner : completedTasks) { - Task task = taskRunner.getTask(); - if (task.isMapRedTask() && !task.isMapRedLocalTask()) { - try { - logEnterExitCounters(task); - - } catch (Exception e) { - SessionState.getConsole().printError("Error in get counters: " + e.toString()); - } - } - } - } - } -} diff --git ql/src/test/queries/clientpositive/hiveprofiler0.q ql/src/test/queries/clientpositive/hiveprofiler0.q deleted file mode 100644 index 23996ec..0000000 --- ql/src/test/queries/clientpositive/hiveprofiler0.q +++ /dev/null @@ -1,6 +0,0 @@ -set hive.exec.operator.hooks=org.apache.hadoop.hive.ql.profiler.HiveProfiler; -set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.HiveProfilerResultsHook; -SET hive.task.progress=true; - -select count(1) from src; -explain select count(1) from src; diff --git ql/src/test/queries/clientpositive/hiveprofiler_script0.q ql/src/test/queries/clientpositive/hiveprofiler_script0.q deleted file mode 100644 index 3ee30d3..0000000 --- ql/src/test/queries/clientpositive/hiveprofiler_script0.q +++ /dev/null @@ -1,15 +0,0 @@ -SET hive.task.progress=true; -set hive.exec.operator.hooks=org.apache.hadoop.hive.ql.profiler.HiveProfiler; -set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.HiveProfilerResultsHook; -ADD FILE src/test/scripts/testgrep; - - --- checking that script operator does not cause NPE --- Derby strangeness is causing the output collector for the Hive Profiler to not get output during DB read - -SELECT TRANSFORM(src.key, src.value) - USING 'testgrep' AS (tkey, tvalue) -FROM src - - - diff --git ql/src/test/queries/clientpositive/hiveprofiler_union0.q ql/src/test/queries/clientpositive/hiveprofiler_union0.q deleted file mode 100644 index 0a94171..0000000 --- ql/src/test/queries/clientpositive/hiveprofiler_union0.q +++ /dev/null @@ -1,19 +0,0 @@ -set hive.exec.operator.hooks=org.apache.hadoop.hive.ql.profiler.HiveProfiler; -set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.HiveProfilerResultsHook; -SET hive.task.progress=true; -FROM ( - FROM src select src.key, src.value WHERE src.key < 100 - UNION ALL - FROM src SELECT src.* WHERE src.key > 100 -) unioninput -SELECT unioninput.*; - -explain - FROM ( - FROM src select src.key, src.value WHERE src.key < 100 - UNION ALL - FROM src SELECT src.* WHERE src.key > 100 - ) unioninput - SELECT unioninput.*; - - diff --git ql/src/test/queries/clientpositive/operatorhook.q ql/src/test/queries/clientpositive/operatorhook.q deleted file mode 100644 index c29ac2e..0000000 --- ql/src/test/queries/clientpositive/operatorhook.q +++ /dev/null @@ -1,6 +0,0 @@ -SET hive.exec.operator.hooks=org.apache.hadoop.hive.ql.exec.TstOperatorHook; -SET hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostTestOperatorHook; -SET hive.exec.mode.local.auto=false; -SET hive.task.progress=true; - -SELECT count(1) FROM src diff --git ql/src/test/results/clientpositive/hiveprofiler0.q.out ql/src/test/results/clientpositive/hiveprofiler0.q.out deleted file mode 100644 index d1777e2..0000000 --- ql/src/test/results/clientpositive/hiveprofiler0.q.out +++ /dev/null @@ -1,65 +0,0 @@ -PREHOOK: query: select count(1) from src -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -GBY_2 ==> RS_3: 1 -TS_0 ==> SEL_1: 500 -SEL_1 ==> GBY_2: 500 -GBY_4 ==> SEL_5: 1 -main() ==> GBY_4: 1 -main() ==> TS_0: 500 -SEL_5 ==> FS_6: 1 -500 -PREHOOK: query: explain select count(1) from src -PREHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION count 1))))) - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Alias -> Map Operator Tree: - src - TableScan - alias: src - Select Operator - Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: - expr: _col0 - type: bigint - Reduce Operator Tree: - Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - mode: mergepartial - outputColumnNames: _col0 - Select Operator - expressions: - expr: _col0 - type: bigint - outputColumnNames: _col0 - File Output Operator - compressed: false - GlobalTableId: 0 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - - Stage: Stage-0 - Fetch Operator - limit: -1 - - diff --git ql/src/test/results/clientpositive/hiveprofiler_script0.q.out ql/src/test/results/clientpositive/hiveprofiler_script0.q.out deleted file mode 100644 index cc03f5d..0000000 --- ql/src/test/results/clientpositive/hiveprofiler_script0.q.out +++ /dev/null @@ -1,18 +0,0 @@ -PREHOOK: query: -- checking that script operator does not cause NPE --- Derby strangeness is causing the output collector for the Hive Profiler to not get output during DB read - -SELECT TRANSFORM(src.key, src.value) - USING 'testgrep' AS (tkey, tvalue) -FROM src -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -103 val_103 -100 val_100 -10 val_10 -103 val_103 -104 val_104 -105 val_105 -104 val_104 -310 val_310 -100 val_100 diff --git ql/src/test/results/clientpositive/hiveprofiler_union0.q.out ql/src/test/results/clientpositive/hiveprofiler_union0.q.out deleted file mode 100644 index ac5c9dc..0000000 --- ql/src/test/results/clientpositive/hiveprofiler_union0.q.out +++ /dev/null @@ -1,598 +0,0 @@ -PREHOOK: query: FROM ( - FROM src select src.key, src.value WHERE src.key < 100 - UNION ALL - FROM src SELECT src.* WHERE src.key > 100 -) unioninput -SELECT unioninput.* -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -SEL_5 ==> UNION_6: 414 -FIL_10 ==> SEL_5: 414 -TS_0 ==> FIL_9: 500 -TS_3 ==> FIL_10: 500 -UNION_6 ==> SEL_7: 498 -SEL_2 ==> UNION_6: 84 -FIL_9 ==> SEL_2: 84 -main() ==> TS_3: 500 -main() ==> TS_0: 500 -SEL_7 ==> FS_8: 498 -238 val_238 -86 val_86 -311 val_311 -27 val_27 -165 val_165 -409 val_409 -255 val_255 -278 val_278 -98 val_98 -484 val_484 -265 val_265 -193 val_193 -401 val_401 -150 val_150 -273 val_273 -224 val_224 -369 val_369 -66 val_66 -128 val_128 -213 val_213 -146 val_146 -406 val_406 -429 val_429 -374 val_374 -152 val_152 -469 val_469 -145 val_145 -495 val_495 -37 val_37 -327 val_327 -281 val_281 -277 val_277 -209 val_209 -15 val_15 -82 val_82 -403 val_403 -166 val_166 -417 val_417 -430 val_430 -252 val_252 -292 val_292 -219 val_219 -287 val_287 -153 val_153 -193 val_193 -338 val_338 -446 val_446 -459 val_459 -394 val_394 -237 val_237 -482 val_482 -174 val_174 -413 val_413 -494 val_494 -207 val_207 -199 val_199 -466 val_466 -208 val_208 -174 val_174 -399 val_399 -396 val_396 -247 val_247 -417 val_417 -489 val_489 -162 val_162 -377 val_377 -397 val_397 -309 val_309 -365 val_365 -266 val_266 -439 val_439 -342 val_342 -367 val_367 -325 val_325 -167 val_167 -195 val_195 -475 val_475 -17 val_17 -113 val_113 -155 val_155 -203 val_203 -339 val_339 -0 val_0 -455 val_455 -128 val_128 -311 val_311 -316 val_316 -57 val_57 -302 val_302 -205 val_205 -149 val_149 -438 val_438 -345 val_345 -129 val_129 -170 val_170 -20 val_20 -489 val_489 -157 val_157 -378 val_378 -221 val_221 -92 val_92 -111 val_111 -47 val_47 -72 val_72 -4 val_4 -280 val_280 -35 val_35 -427 val_427 -277 val_277 -208 val_208 -356 val_356 -399 val_399 -169 val_169 -382 val_382 -498 val_498 -125 val_125 -386 val_386 -437 val_437 -469 val_469 -192 val_192 -286 val_286 -187 val_187 -176 val_176 -54 val_54 -459 val_459 -51 val_51 -138 val_138 -103 val_103 -239 val_239 -213 val_213 -216 val_216 -430 val_430 -278 val_278 -176 val_176 -289 val_289 -221 val_221 -65 val_65 -318 val_318 -332 val_332 -311 val_311 -275 val_275 -137 val_137 -241 val_241 -83 val_83 -333 val_333 -180 val_180 -284 val_284 -12 val_12 -230 val_230 -181 val_181 -67 val_67 -260 val_260 -404 val_404 -384 val_384 -489 val_489 -353 val_353 -373 val_373 -272 val_272 -138 val_138 -217 val_217 -84 val_84 -348 val_348 -466 val_466 -58 val_58 -8 val_8 -411 val_411 -230 val_230 -208 val_208 -348 val_348 -24 val_24 -463 val_463 -431 val_431 -179 val_179 -172 val_172 -42 val_42 -129 val_129 -158 val_158 -119 val_119 -496 val_496 -0 val_0 -322 val_322 -197 val_197 -468 val_468 -393 val_393 -454 val_454 -298 val_298 -199 val_199 -191 val_191 -418 val_418 -96 val_96 -26 val_26 -165 val_165 -327 val_327 -230 val_230 -205 val_205 -120 val_120 -131 val_131 -51 val_51 -404 val_404 -43 val_43 -436 val_436 -156 val_156 -469 val_469 -468 val_468 -308 val_308 -95 val_95 -196 val_196 -288 val_288 -481 val_481 -457 val_457 -98 val_98 -282 val_282 -197 val_197 -187 val_187 -318 val_318 -318 val_318 -409 val_409 -470 val_470 -137 val_137 -369 val_369 -316 val_316 -169 val_169 -413 val_413 -85 val_85 -77 val_77 -0 val_0 -490 val_490 -87 val_87 -364 val_364 -179 val_179 -118 val_118 -134 val_134 -395 val_395 -282 val_282 -138 val_138 -238 val_238 -419 val_419 -15 val_15 -118 val_118 -72 val_72 -90 val_90 -307 val_307 -19 val_19 -435 val_435 -10 val_10 -277 val_277 -273 val_273 -306 val_306 -224 val_224 -309 val_309 -389 val_389 -327 val_327 -242 val_242 -369 val_369 -392 val_392 -272 val_272 -331 val_331 -401 val_401 -242 val_242 -452 val_452 -177 val_177 -226 val_226 -5 val_5 -497 val_497 -402 val_402 -396 val_396 -317 val_317 -395 val_395 -58 val_58 -35 val_35 -336 val_336 -95 val_95 -11 val_11 -168 val_168 -34 val_34 -229 val_229 -233 val_233 -143 val_143 -472 val_472 -322 val_322 -498 val_498 -160 val_160 -195 val_195 -42 val_42 -321 val_321 -430 val_430 -119 val_119 -489 val_489 -458 val_458 -78 val_78 -76 val_76 -41 val_41 -223 val_223 -492 val_492 -149 val_149 -449 val_449 -218 val_218 -228 val_228 -138 val_138 -453 val_453 -30 val_30 -209 val_209 -64 val_64 -468 val_468 -76 val_76 -74 val_74 -342 val_342 -69 val_69 -230 val_230 -33 val_33 -368 val_368 -103 val_103 -296 val_296 -113 val_113 -216 val_216 -367 val_367 -344 val_344 -167 val_167 -274 val_274 -219 val_219 -239 val_239 -485 val_485 -116 val_116 -223 val_223 -256 val_256 -263 val_263 -70 val_70 -487 val_487 -480 val_480 -401 val_401 -288 val_288 -191 val_191 -5 val_5 -244 val_244 -438 val_438 -128 val_128 -467 val_467 -432 val_432 -202 val_202 -316 val_316 -229 val_229 -469 val_469 -463 val_463 -280 val_280 -2 val_2 -35 val_35 -283 val_283 -331 val_331 -235 val_235 -80 val_80 -44 val_44 -193 val_193 -321 val_321 -335 val_335 -104 val_104 -466 val_466 -366 val_366 -175 val_175 -403 val_403 -483 val_483 -53 val_53 -105 val_105 -257 val_257 -406 val_406 -409 val_409 -190 val_190 -406 val_406 -401 val_401 -114 val_114 -258 val_258 -90 val_90 -203 val_203 -262 val_262 -348 val_348 -424 val_424 -12 val_12 -396 val_396 -201 val_201 -217 val_217 -164 val_164 -431 val_431 -454 val_454 -478 val_478 -298 val_298 -125 val_125 -431 val_431 -164 val_164 -424 val_424 -187 val_187 -382 val_382 -5 val_5 -70 val_70 -397 val_397 -480 val_480 -291 val_291 -24 val_24 -351 val_351 -255 val_255 -104 val_104 -70 val_70 -163 val_163 -438 val_438 -119 val_119 -414 val_414 -200 val_200 -491 val_491 -237 val_237 -439 val_439 -360 val_360 -248 val_248 -479 val_479 -305 val_305 -417 val_417 -199 val_199 -444 val_444 -120 val_120 -429 val_429 -169 val_169 -443 val_443 -323 val_323 -325 val_325 -277 val_277 -230 val_230 -478 val_478 -178 val_178 -468 val_468 -310 val_310 -317 val_317 -333 val_333 -493 val_493 -460 val_460 -207 val_207 -249 val_249 -265 val_265 -480 val_480 -83 val_83 -136 val_136 -353 val_353 -172 val_172 -214 val_214 -462 val_462 -233 val_233 -406 val_406 -133 val_133 -175 val_175 -189 val_189 -454 val_454 -375 val_375 -401 val_401 -421 val_421 -407 val_407 -384 val_384 -256 val_256 -26 val_26 -134 val_134 -67 val_67 -384 val_384 -379 val_379 -18 val_18 -462 val_462 -492 val_492 -298 val_298 -9 val_9 -341 val_341 -498 val_498 -146 val_146 -458 val_458 -362 val_362 -186 val_186 -285 val_285 -348 val_348 -167 val_167 -18 val_18 -273 val_273 -183 val_183 -281 val_281 -344 val_344 -97 val_97 -469 val_469 -315 val_315 -84 val_84 -28 val_28 -37 val_37 -448 val_448 -152 val_152 -348 val_348 -307 val_307 -194 val_194 -414 val_414 -477 val_477 -222 val_222 -126 val_126 -90 val_90 -169 val_169 -403 val_403 -400 val_400 -200 val_200 -97 val_97 -PREHOOK: query: explain - FROM ( - FROM src select src.key, src.value WHERE src.key < 100 - UNION ALL - FROM src SELECT src.* WHERE src.key > 100 - ) unioninput - SELECT unioninput.* -PREHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) value))) (TOK_WHERE (< (. (TOK_TABLE_OR_COL src) key) 100)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME src)))) (TOK_WHERE (> (. (TOK_TABLE_OR_COL src) key) 100))))) unioninput)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME unioninput)))))) - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Alias -> Map Operator Tree: - null-subquery1:unioninput-subquery1:src - TableScan - alias: src - Filter Operator - predicate: - expr: (key < 100.0) - type: boolean - Select Operator - expressions: - expr: key - type: string - expr: value - type: string - outputColumnNames: _col0, _col1 - Union - Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: string - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 0 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - null-subquery2:unioninput-subquery2:src - TableScan - alias: src - Filter Operator - predicate: - expr: (key > 100.0) - type: boolean - Select Operator - expressions: - expr: key - type: string - expr: value - type: string - outputColumnNames: _col0, _col1 - Union - Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: string - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 0 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - - Stage: Stage-0 - Fetch Operator - limit: -1 - - diff --git ql/src/test/results/clientpositive/operatorhook.q.out ql/src/test/results/clientpositive/operatorhook.q.out deleted file mode 100644 index 541da0a..0000000 --- ql/src/test/results/clientpositive/operatorhook.q.out +++ /dev/null @@ -1,7 +0,0 @@ -PREHOOK: query: SELECT count(1) FROM src -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -TEST_OPERATOR_HOOK_EXIT: 7508 -TEST_OPERATOR_HOOK_ENTER: 7508 -500